From 8a7b8f0676da54e60934ea391d2009875478aab2 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Wed, 5 Apr 2017 08:26:08 +0200 Subject: [PATCH 01/87] Update shas. --- buildSrc/version.properties | 2 +- core/licenses/lucene-analyzers-common-6.5.0.jar.sha1 | 1 - .../lucene-analyzers-common-7.0.0-snapshot-05749d0.jar.sha1 | 1 + core/licenses/lucene-backward-codecs-6.5.0.jar.sha1 | 1 - .../lucene-backward-codecs-7.0.0-snapshot-05749d0.jar.sha1 | 1 + core/licenses/lucene-core-6.5.0.jar.sha1 | 1 - core/licenses/lucene-core-7.0.0-snapshot-05749d0.jar.sha1 | 1 + core/licenses/lucene-grouping-6.5.0.jar.sha1 | 1 - core/licenses/lucene-grouping-7.0.0-snapshot-05749d0.jar.sha1 | 1 + core/licenses/lucene-highlighter-6.5.0.jar.sha1 | 1 - .../lucene-highlighter-7.0.0-snapshot-05749d0.jar.sha1 | 1 + core/licenses/lucene-join-6.5.0.jar.sha1 | 1 - core/licenses/lucene-join-7.0.0-snapshot-05749d0.jar.sha1 | 1 + core/licenses/lucene-memory-6.5.0.jar.sha1 | 1 - core/licenses/lucene-memory-7.0.0-snapshot-05749d0.jar.sha1 | 1 + core/licenses/lucene-misc-6.5.0.jar.sha1 | 1 - core/licenses/lucene-misc-7.0.0-snapshot-05749d0.jar.sha1 | 1 + core/licenses/lucene-queries-6.5.0.jar.sha1 | 1 - core/licenses/lucene-queries-7.0.0-snapshot-05749d0.jar.sha1 | 1 + core/licenses/lucene-queryparser-6.5.0.jar.sha1 | 1 - .../lucene-queryparser-7.0.0-snapshot-05749d0.jar.sha1 | 1 + core/licenses/lucene-sandbox-6.5.0.jar.sha1 | 1 - core/licenses/lucene-sandbox-7.0.0-snapshot-05749d0.jar.sha1 | 1 + core/licenses/lucene-spatial-6.5.0.jar.sha1 | 1 - core/licenses/lucene-spatial-7.0.0-snapshot-05749d0.jar.sha1 | 1 + core/licenses/lucene-spatial-extras-6.5.0.jar.sha1 | 1 - .../lucene-spatial-extras-7.0.0-snapshot-05749d0.jar.sha1 | 1 + core/licenses/lucene-spatial3d-6.5.0.jar.sha1 | 1 - .../licenses/lucene-spatial3d-7.0.0-snapshot-05749d0.jar.sha1 | 1 + core/licenses/lucene-suggest-6.5.0.jar.sha1 | 1 - core/licenses/lucene-suggest-7.0.0-snapshot-05749d0.jar.sha1 | 1 + .../resources/org/elasticsearch/bootstrap/security.policy | 4 ++-- .../org/elasticsearch/bootstrap/test-framework.policy | 2 +- .../licenses/lucene-expressions-6.5.0.jar.sha1 | 1 - .../lucene-expressions-7.0.0-snapshot-05749d0.jar.sha1 | 1 + .../analysis-icu/licenses/lucene-analyzers-icu-6.5.0.jar.sha1 | 1 - .../lucene-analyzers-icu-7.0.0-snapshot-05749d0.jar.sha1 | 1 + .../licenses/lucene-analyzers-kuromoji-6.5.0.jar.sha1 | 1 - .../lucene-analyzers-kuromoji-7.0.0-snapshot-05749d0.jar.sha1 | 1 + .../licenses/lucene-analyzers-phonetic-6.5.0.jar.sha1 | 1 - .../lucene-analyzers-phonetic-7.0.0-snapshot-05749d0.jar.sha1 | 1 + .../licenses/lucene-analyzers-smartcn-6.5.0.jar.sha1 | 1 - .../lucene-analyzers-smartcn-7.0.0-snapshot-05749d0.jar.sha1 | 1 + .../licenses/lucene-analyzers-stempel-6.5.0.jar.sha1 | 1 - .../lucene-analyzers-stempel-7.0.0-snapshot-05749d0.jar.sha1 | 1 + .../licenses/lucene-analyzers-morfologik-6.5.0.jar.sha1 | 1 - ...ucene-analyzers-morfologik-7.0.0-snapshot-05749d0.jar.sha1 | 1 + 47 files changed, 26 insertions(+), 26 deletions(-) delete mode 100644 core/licenses/lucene-analyzers-common-6.5.0.jar.sha1 create mode 100644 core/licenses/lucene-analyzers-common-7.0.0-snapshot-05749d0.jar.sha1 delete mode 100644 core/licenses/lucene-backward-codecs-6.5.0.jar.sha1 create mode 100644 core/licenses/lucene-backward-codecs-7.0.0-snapshot-05749d0.jar.sha1 delete mode 100644 core/licenses/lucene-core-6.5.0.jar.sha1 create mode 100644 core/licenses/lucene-core-7.0.0-snapshot-05749d0.jar.sha1 delete mode 100644 core/licenses/lucene-grouping-6.5.0.jar.sha1 create mode 100644 core/licenses/lucene-grouping-7.0.0-snapshot-05749d0.jar.sha1 delete mode 100644 core/licenses/lucene-highlighter-6.5.0.jar.sha1 create mode 100644 core/licenses/lucene-highlighter-7.0.0-snapshot-05749d0.jar.sha1 delete mode 100644 core/licenses/lucene-join-6.5.0.jar.sha1 create mode 100644 core/licenses/lucene-join-7.0.0-snapshot-05749d0.jar.sha1 delete mode 100644 core/licenses/lucene-memory-6.5.0.jar.sha1 create mode 100644 core/licenses/lucene-memory-7.0.0-snapshot-05749d0.jar.sha1 delete mode 100644 core/licenses/lucene-misc-6.5.0.jar.sha1 create mode 100644 core/licenses/lucene-misc-7.0.0-snapshot-05749d0.jar.sha1 delete mode 100644 core/licenses/lucene-queries-6.5.0.jar.sha1 create mode 100644 core/licenses/lucene-queries-7.0.0-snapshot-05749d0.jar.sha1 delete mode 100644 core/licenses/lucene-queryparser-6.5.0.jar.sha1 create mode 100644 core/licenses/lucene-queryparser-7.0.0-snapshot-05749d0.jar.sha1 delete mode 100644 core/licenses/lucene-sandbox-6.5.0.jar.sha1 create mode 100644 core/licenses/lucene-sandbox-7.0.0-snapshot-05749d0.jar.sha1 delete mode 100644 core/licenses/lucene-spatial-6.5.0.jar.sha1 create mode 100644 core/licenses/lucene-spatial-7.0.0-snapshot-05749d0.jar.sha1 delete mode 100644 core/licenses/lucene-spatial-extras-6.5.0.jar.sha1 create mode 100644 core/licenses/lucene-spatial-extras-7.0.0-snapshot-05749d0.jar.sha1 delete mode 100644 core/licenses/lucene-spatial3d-6.5.0.jar.sha1 create mode 100644 core/licenses/lucene-spatial3d-7.0.0-snapshot-05749d0.jar.sha1 delete mode 100644 core/licenses/lucene-suggest-6.5.0.jar.sha1 create mode 100644 core/licenses/lucene-suggest-7.0.0-snapshot-05749d0.jar.sha1 delete mode 100644 modules/lang-expression/licenses/lucene-expressions-6.5.0.jar.sha1 create mode 100644 modules/lang-expression/licenses/lucene-expressions-7.0.0-snapshot-05749d0.jar.sha1 delete mode 100644 plugins/analysis-icu/licenses/lucene-analyzers-icu-6.5.0.jar.sha1 create mode 100644 plugins/analysis-icu/licenses/lucene-analyzers-icu-7.0.0-snapshot-05749d0.jar.sha1 delete mode 100644 plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-6.5.0.jar.sha1 create mode 100644 plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-7.0.0-snapshot-05749d0.jar.sha1 delete mode 100644 plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-6.5.0.jar.sha1 create mode 100644 plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-7.0.0-snapshot-05749d0.jar.sha1 delete mode 100644 plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-6.5.0.jar.sha1 create mode 100644 plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-7.0.0-snapshot-05749d0.jar.sha1 delete mode 100644 plugins/analysis-stempel/licenses/lucene-analyzers-stempel-6.5.0.jar.sha1 create mode 100644 plugins/analysis-stempel/licenses/lucene-analyzers-stempel-7.0.0-snapshot-05749d0.jar.sha1 delete mode 100644 plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-6.5.0.jar.sha1 create mode 100644 plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-7.0.0-snapshot-05749d0.jar.sha1 diff --git a/buildSrc/version.properties b/buildSrc/version.properties index cea96db283d15..fdf0d7c034e81 100644 --- a/buildSrc/version.properties +++ b/buildSrc/version.properties @@ -1,6 +1,6 @@ # When updating elasticsearch, please update 'rest' version in core/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy elasticsearch = 6.0.0-alpha1 -lucene = 6.5.0 +lucene = 7.0.0-snapshot-05749d0 # optional dependencies spatial4j = 0.6 diff --git a/core/licenses/lucene-analyzers-common-6.5.0.jar.sha1 b/core/licenses/lucene-analyzers-common-6.5.0.jar.sha1 deleted file mode 100644 index 77d21a23774e3..0000000000000 --- a/core/licenses/lucene-analyzers-common-6.5.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -3989779b05ecd0ace6affe19223b1c27156604f1 \ No newline at end of file diff --git a/core/licenses/lucene-analyzers-common-7.0.0-snapshot-05749d0.jar.sha1 b/core/licenses/lucene-analyzers-common-7.0.0-snapshot-05749d0.jar.sha1 new file mode 100644 index 0000000000000..37783c4b629fb --- /dev/null +++ b/core/licenses/lucene-analyzers-common-7.0.0-snapshot-05749d0.jar.sha1 @@ -0,0 +1 @@ +364580e75893da429db6314cbbe3810afc30dfdc \ No newline at end of file diff --git a/core/licenses/lucene-backward-codecs-6.5.0.jar.sha1 b/core/licenses/lucene-backward-codecs-6.5.0.jar.sha1 deleted file mode 100644 index 9eaff80ac08e0..0000000000000 --- a/core/licenses/lucene-backward-codecs-6.5.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -6a8660e7133f357ef40d9cac26316ccd9937a2eb \ No newline at end of file diff --git a/core/licenses/lucene-backward-codecs-7.0.0-snapshot-05749d0.jar.sha1 b/core/licenses/lucene-backward-codecs-7.0.0-snapshot-05749d0.jar.sha1 new file mode 100644 index 0000000000000..a5faad824dc2d --- /dev/null +++ b/core/licenses/lucene-backward-codecs-7.0.0-snapshot-05749d0.jar.sha1 @@ -0,0 +1 @@ +af62e6aad7490205e350d986a53b495c384df5ac \ No newline at end of file diff --git a/core/licenses/lucene-core-6.5.0.jar.sha1 b/core/licenses/lucene-core-6.5.0.jar.sha1 deleted file mode 100644 index 7af91ec15bf1e..0000000000000 --- a/core/licenses/lucene-core-6.5.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -ff176c9bde4228b43827849f5d2ff2e2717e3297 \ No newline at end of file diff --git a/core/licenses/lucene-core-7.0.0-snapshot-05749d0.jar.sha1 b/core/licenses/lucene-core-7.0.0-snapshot-05749d0.jar.sha1 new file mode 100644 index 0000000000000..08054972a2a1c --- /dev/null +++ b/core/licenses/lucene-core-7.0.0-snapshot-05749d0.jar.sha1 @@ -0,0 +1 @@ +31f5d32778e1dbcc909f4245cdfd691d2f16f7d5 \ No newline at end of file diff --git a/core/licenses/lucene-grouping-6.5.0.jar.sha1 b/core/licenses/lucene-grouping-6.5.0.jar.sha1 deleted file mode 100644 index 08ccc2cd08690..0000000000000 --- a/core/licenses/lucene-grouping-6.5.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -10d2e5b36f460527ac9b948be0ec3077bde5b0ca \ No newline at end of file diff --git a/core/licenses/lucene-grouping-7.0.0-snapshot-05749d0.jar.sha1 b/core/licenses/lucene-grouping-7.0.0-snapshot-05749d0.jar.sha1 new file mode 100644 index 0000000000000..e4d38ee0e9995 --- /dev/null +++ b/core/licenses/lucene-grouping-7.0.0-snapshot-05749d0.jar.sha1 @@ -0,0 +1 @@ +76721f38a42d18694be50d765dc34e0aa8fd0f7d \ No newline at end of file diff --git a/core/licenses/lucene-highlighter-6.5.0.jar.sha1 b/core/licenses/lucene-highlighter-6.5.0.jar.sha1 deleted file mode 100644 index a8069723f1640..0000000000000 --- a/core/licenses/lucene-highlighter-6.5.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -0019bb6a631ea0123e8e553b0510fa81c9d3c3eb \ No newline at end of file diff --git a/core/licenses/lucene-highlighter-7.0.0-snapshot-05749d0.jar.sha1 b/core/licenses/lucene-highlighter-7.0.0-snapshot-05749d0.jar.sha1 new file mode 100644 index 0000000000000..626d86e955f29 --- /dev/null +++ b/core/licenses/lucene-highlighter-7.0.0-snapshot-05749d0.jar.sha1 @@ -0,0 +1 @@ +5bb097fe7d91ed39544b8c2042e7f9de0a4cb896 \ No newline at end of file diff --git a/core/licenses/lucene-join-6.5.0.jar.sha1 b/core/licenses/lucene-join-6.5.0.jar.sha1 deleted file mode 100644 index cbad6199d76b5..0000000000000 --- a/core/licenses/lucene-join-6.5.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -dad85baba266793b9ceb80a9b08c4ee9838e09df \ No newline at end of file diff --git a/core/licenses/lucene-join-7.0.0-snapshot-05749d0.jar.sha1 b/core/licenses/lucene-join-7.0.0-snapshot-05749d0.jar.sha1 new file mode 100644 index 0000000000000..9505d2beb9f75 --- /dev/null +++ b/core/licenses/lucene-join-7.0.0-snapshot-05749d0.jar.sha1 @@ -0,0 +1 @@ +95ddeed8524ea4feaca455b962ca1883735d241f \ No newline at end of file diff --git a/core/licenses/lucene-memory-6.5.0.jar.sha1 b/core/licenses/lucene-memory-6.5.0.jar.sha1 deleted file mode 100644 index 5f22c0d9cfe3b..0000000000000 --- a/core/licenses/lucene-memory-6.5.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -938f9f7efe8a403fd57c99aedd75d040d9caa896 \ No newline at end of file diff --git a/core/licenses/lucene-memory-7.0.0-snapshot-05749d0.jar.sha1 b/core/licenses/lucene-memory-7.0.0-snapshot-05749d0.jar.sha1 new file mode 100644 index 0000000000000..8e5290de79fdc --- /dev/null +++ b/core/licenses/lucene-memory-7.0.0-snapshot-05749d0.jar.sha1 @@ -0,0 +1 @@ +3057c32455677c4656f681757a696b8eaac0f36f \ No newline at end of file diff --git a/core/licenses/lucene-misc-6.5.0.jar.sha1 b/core/licenses/lucene-misc-6.5.0.jar.sha1 deleted file mode 100644 index 2b405d7f16aa9..0000000000000 --- a/core/licenses/lucene-misc-6.5.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -afdff39ecb30f6e2c6f056a5bdfcb13d928a25af \ No newline at end of file diff --git a/core/licenses/lucene-misc-7.0.0-snapshot-05749d0.jar.sha1 b/core/licenses/lucene-misc-7.0.0-snapshot-05749d0.jar.sha1 new file mode 100644 index 0000000000000..190bf1549ad4a --- /dev/null +++ b/core/licenses/lucene-misc-7.0.0-snapshot-05749d0.jar.sha1 @@ -0,0 +1 @@ +0ef5a3ba35c6a3c3511407918d9d499ce55ea2ff \ No newline at end of file diff --git a/core/licenses/lucene-queries-6.5.0.jar.sha1 b/core/licenses/lucene-queries-6.5.0.jar.sha1 deleted file mode 100644 index 9a046ce204f08..0000000000000 --- a/core/licenses/lucene-queries-6.5.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -8e3971a008070712d57b59cf1f7b44c0d9d3df25 \ No newline at end of file diff --git a/core/licenses/lucene-queries-7.0.0-snapshot-05749d0.jar.sha1 b/core/licenses/lucene-queries-7.0.0-snapshot-05749d0.jar.sha1 new file mode 100644 index 0000000000000..56010255eac1a --- /dev/null +++ b/core/licenses/lucene-queries-7.0.0-snapshot-05749d0.jar.sha1 @@ -0,0 +1 @@ +7f8b9c2d214cc60fbc0e1bf8d6a140d514ee8100 \ No newline at end of file diff --git a/core/licenses/lucene-queryparser-6.5.0.jar.sha1 b/core/licenses/lucene-queryparser-6.5.0.jar.sha1 deleted file mode 100644 index 3136885ab92ca..0000000000000 --- a/core/licenses/lucene-queryparser-6.5.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -225b904edf91ccdffffa398e1924ebadd5677c09 \ No newline at end of file diff --git a/core/licenses/lucene-queryparser-7.0.0-snapshot-05749d0.jar.sha1 b/core/licenses/lucene-queryparser-7.0.0-snapshot-05749d0.jar.sha1 new file mode 100644 index 0000000000000..e630517801ef3 --- /dev/null +++ b/core/licenses/lucene-queryparser-7.0.0-snapshot-05749d0.jar.sha1 @@ -0,0 +1 @@ +f288aa637a02b074f9fcbff25debe37549a7dfd3 \ No newline at end of file diff --git a/core/licenses/lucene-sandbox-6.5.0.jar.sha1 b/core/licenses/lucene-sandbox-6.5.0.jar.sha1 deleted file mode 100644 index e3787e336dfbf..0000000000000 --- a/core/licenses/lucene-sandbox-6.5.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -5c994fc5dc4f37133a861571211303d81c5d51ff \ No newline at end of file diff --git a/core/licenses/lucene-sandbox-7.0.0-snapshot-05749d0.jar.sha1 b/core/licenses/lucene-sandbox-7.0.0-snapshot-05749d0.jar.sha1 new file mode 100644 index 0000000000000..c96e2aa6d7f3a --- /dev/null +++ b/core/licenses/lucene-sandbox-7.0.0-snapshot-05749d0.jar.sha1 @@ -0,0 +1 @@ +1da8f0a2f4ec16b20300401357396c832ef9b501 \ No newline at end of file diff --git a/core/licenses/lucene-spatial-6.5.0.jar.sha1 b/core/licenses/lucene-spatial-6.5.0.jar.sha1 deleted file mode 100644 index cbadbfc42d75f..0000000000000 --- a/core/licenses/lucene-spatial-6.5.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -553b7b13bef994f14076a85557df03cad67322e9 \ No newline at end of file diff --git a/core/licenses/lucene-spatial-7.0.0-snapshot-05749d0.jar.sha1 b/core/licenses/lucene-spatial-7.0.0-snapshot-05749d0.jar.sha1 new file mode 100644 index 0000000000000..355ecc2b23116 --- /dev/null +++ b/core/licenses/lucene-spatial-7.0.0-snapshot-05749d0.jar.sha1 @@ -0,0 +1 @@ +4a9780f5a2446d8d4f92919ce1e8e73d73fa2698 \ No newline at end of file diff --git a/core/licenses/lucene-spatial-extras-6.5.0.jar.sha1 b/core/licenses/lucene-spatial-extras-6.5.0.jar.sha1 deleted file mode 100644 index f2ad71855f292..0000000000000 --- a/core/licenses/lucene-spatial-extras-6.5.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -73deae791d861820974600705ba06e9f801cbe56 \ No newline at end of file diff --git a/core/licenses/lucene-spatial-extras-7.0.0-snapshot-05749d0.jar.sha1 b/core/licenses/lucene-spatial-extras-7.0.0-snapshot-05749d0.jar.sha1 new file mode 100644 index 0000000000000..2d94742bc600c --- /dev/null +++ b/core/licenses/lucene-spatial-extras-7.0.0-snapshot-05749d0.jar.sha1 @@ -0,0 +1 @@ +6bb7d4f0270eb69ff01d13cc34ac75fd4451ec9e \ No newline at end of file diff --git a/core/licenses/lucene-spatial3d-6.5.0.jar.sha1 b/core/licenses/lucene-spatial3d-6.5.0.jar.sha1 deleted file mode 100644 index 8fca9ac1ebc68..0000000000000 --- a/core/licenses/lucene-spatial3d-6.5.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -c2aad69500dac79338ef45f570cab47bec3d2724 \ No newline at end of file diff --git a/core/licenses/lucene-spatial3d-7.0.0-snapshot-05749d0.jar.sha1 b/core/licenses/lucene-spatial3d-7.0.0-snapshot-05749d0.jar.sha1 new file mode 100644 index 0000000000000..49b43a35c4ce2 --- /dev/null +++ b/core/licenses/lucene-spatial3d-7.0.0-snapshot-05749d0.jar.sha1 @@ -0,0 +1 @@ +fbf9869d1820bf7a6756e05f02467b140b30ff12 \ No newline at end of file diff --git a/core/licenses/lucene-suggest-6.5.0.jar.sha1 b/core/licenses/lucene-suggest-6.5.0.jar.sha1 deleted file mode 100644 index 62764fbbc3270..0000000000000 --- a/core/licenses/lucene-suggest-6.5.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -acf211f2bf901dfc8155a46c5a42c5650edf74ef \ No newline at end of file diff --git a/core/licenses/lucene-suggest-7.0.0-snapshot-05749d0.jar.sha1 b/core/licenses/lucene-suggest-7.0.0-snapshot-05749d0.jar.sha1 new file mode 100644 index 0000000000000..9f39e5dd8f61c --- /dev/null +++ b/core/licenses/lucene-suggest-7.0.0-snapshot-05749d0.jar.sha1 @@ -0,0 +1 @@ +af8c10761e1b7186882613dce5f714c888e50371 \ No newline at end of file diff --git a/core/src/main/resources/org/elasticsearch/bootstrap/security.policy b/core/src/main/resources/org/elasticsearch/bootstrap/security.policy index 5c9c56de18250..fa8099ead2336 100644 --- a/core/src/main/resources/org/elasticsearch/bootstrap/security.policy +++ b/core/src/main/resources/org/elasticsearch/bootstrap/security.policy @@ -31,7 +31,7 @@ grant codeBase "${codebase.securesm-1.1.jar}" { //// Very special jar permissions: //// These are dangerous permissions that we don't want to grant to everything. -grant codeBase "${codebase.lucene-core-6.5.0.jar}" { +grant codeBase "${codebase.lucene-core-7.0.0-snapshot-05749d0.jar}" { // needed to allow MMapDirectory's "unmap hack" (die unmap hack, die) // java 8 package permission java.lang.RuntimePermission "accessClassInPackage.sun.misc"; @@ -42,7 +42,7 @@ grant codeBase "${codebase.lucene-core-6.5.0.jar}" { permission java.lang.RuntimePermission "accessDeclaredMembers"; }; -grant codeBase "${codebase.lucene-misc-6.5.0.jar}" { +grant codeBase "${codebase.lucene-misc-7.0.0-snapshot-05749d0.jar}" { // needed to allow shard shrinking to use hard-links if possible via lucenes HardlinkCopyDirectoryWrapper permission java.nio.file.LinkPermission "hard"; }; diff --git a/core/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy b/core/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy index 2c39ccb350ea3..8c9655c89e66b 100644 --- a/core/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy +++ b/core/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy @@ -33,7 +33,7 @@ grant codeBase "${codebase.securemock-1.2.jar}" { permission java.lang.reflect.ReflectPermission "suppressAccessChecks"; }; -grant codeBase "${codebase.lucene-test-framework-6.5.0.jar}" { +grant codeBase "${codebase.lucene-test-framework-7.0.0-snapshot-05749d0.jar}" { // needed by RamUsageTester permission java.lang.reflect.ReflectPermission "suppressAccessChecks"; // needed for testing hardlinks in StoreRecoveryTests since we install MockFS diff --git a/modules/lang-expression/licenses/lucene-expressions-6.5.0.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-6.5.0.jar.sha1 deleted file mode 100644 index 10514bb3d1f7e..0000000000000 --- a/modules/lang-expression/licenses/lucene-expressions-6.5.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -5dfd44932fc77187a233a1cbf228c1a96ac8924f \ No newline at end of file diff --git a/modules/lang-expression/licenses/lucene-expressions-7.0.0-snapshot-05749d0.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-7.0.0-snapshot-05749d0.jar.sha1 new file mode 100644 index 0000000000000..fa698133e4761 --- /dev/null +++ b/modules/lang-expression/licenses/lucene-expressions-7.0.0-snapshot-05749d0.jar.sha1 @@ -0,0 +1 @@ +28c9991d67a24c4a72103cbdcaa1bd560adc67d4 \ No newline at end of file diff --git a/plugins/analysis-icu/licenses/lucene-analyzers-icu-6.5.0.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analyzers-icu-6.5.0.jar.sha1 deleted file mode 100644 index 95df77a75218a..0000000000000 --- a/plugins/analysis-icu/licenses/lucene-analyzers-icu-6.5.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -3a71465f63887f871bc377d87a0838c29b0a857d \ No newline at end of file diff --git a/plugins/analysis-icu/licenses/lucene-analyzers-icu-7.0.0-snapshot-05749d0.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analyzers-icu-7.0.0-snapshot-05749d0.jar.sha1 new file mode 100644 index 0000000000000..50a81a3345b83 --- /dev/null +++ b/plugins/analysis-icu/licenses/lucene-analyzers-icu-7.0.0-snapshot-05749d0.jar.sha1 @@ -0,0 +1 @@ +c71806331eb8c6f690196eac4ec7f038bc027512 \ No newline at end of file diff --git a/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-6.5.0.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-6.5.0.jar.sha1 deleted file mode 100644 index 0c928699fc67d..0000000000000 --- a/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-6.5.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -03353b0d030f6d5a63c4c0d5b64c770f5ba9d829 \ No newline at end of file diff --git a/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-7.0.0-snapshot-05749d0.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-7.0.0-snapshot-05749d0.jar.sha1 new file mode 100644 index 0000000000000..93290a8ed304e --- /dev/null +++ b/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-7.0.0-snapshot-05749d0.jar.sha1 @@ -0,0 +1 @@ +fcc86339e16d85728f2becc2b404aeb69c2fd9fc \ No newline at end of file diff --git a/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-6.5.0.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-6.5.0.jar.sha1 deleted file mode 100644 index ba2bee28476bd..0000000000000 --- a/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-6.5.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -77ce4fb8c62688d8a094f08a07685c464ec46345 \ No newline at end of file diff --git a/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-7.0.0-snapshot-05749d0.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-7.0.0-snapshot-05749d0.jar.sha1 new file mode 100644 index 0000000000000..191344494dc03 --- /dev/null +++ b/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-7.0.0-snapshot-05749d0.jar.sha1 @@ -0,0 +1 @@ +6f3abd78bae787482b256e7629ee6c765546d476 \ No newline at end of file diff --git a/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-6.5.0.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-6.5.0.jar.sha1 deleted file mode 100644 index 0a0ae4cf4011b..0000000000000 --- a/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-6.5.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -60a780d900e48b0cead42d82fe405ad54bd658c3 \ No newline at end of file diff --git a/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-7.0.0-snapshot-05749d0.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-7.0.0-snapshot-05749d0.jar.sha1 new file mode 100644 index 0000000000000..6c9093a23d9c0 --- /dev/null +++ b/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-7.0.0-snapshot-05749d0.jar.sha1 @@ -0,0 +1 @@ +0311b02c31c634cfefaf51b551edb896d73840e2 \ No newline at end of file diff --git a/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-6.5.0.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-6.5.0.jar.sha1 deleted file mode 100644 index 9a1387fa22fdd..0000000000000 --- a/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-6.5.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -894c42c011d291e72d14db660499c75281de9efd \ No newline at end of file diff --git a/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-7.0.0-snapshot-05749d0.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-7.0.0-snapshot-05749d0.jar.sha1 new file mode 100644 index 0000000000000..501140d4149bb --- /dev/null +++ b/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-7.0.0-snapshot-05749d0.jar.sha1 @@ -0,0 +1 @@ +d511a9ef81c2f80361d2f86f8848d6cd504b390e \ No newline at end of file diff --git a/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-6.5.0.jar.sha1 b/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-6.5.0.jar.sha1 deleted file mode 100644 index 89a0283d52e7f..0000000000000 --- a/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-6.5.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -72f0172cf947ab563a7c8166855cf7cbdfe33136 \ No newline at end of file diff --git a/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-7.0.0-snapshot-05749d0.jar.sha1 b/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-7.0.0-snapshot-05749d0.jar.sha1 new file mode 100644 index 0000000000000..f21f6e197f440 --- /dev/null +++ b/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-7.0.0-snapshot-05749d0.jar.sha1 @@ -0,0 +1 @@ +7f0157bf06682dda0d0e0a2120403099ff9f8a36 \ No newline at end of file From bd09d8aa13e4c9727c5b8493ba6489eadb0a19ad Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Wed, 5 Apr 2017 09:44:31 +0200 Subject: [PATCH 02/87] Start migrating fielddata to iterator APIs. --- .../index/fielddata/FieldData.java | 385 +++++++++--------- .../index/fielddata/GeoPointValues.java | 14 +- .../index/fielddata/MultiGeoPointValues.java | 20 +- .../index/fielddata/NumericDoubleValues.java | 69 +++- .../SingletonMultiGeoPointValues.java | 34 +- .../SingletonSortedBinaryDocValues.java | 32 +- .../SingletonSortedNumericDoubleValues.java | 54 +-- .../fielddata/SortedBinaryDocValues.java | 25 +- .../fielddata/SortedNumericDoubleValues.java | 35 +- .../fielddata/SortingBinaryDocValues.java | 9 +- 10 files changed, 348 insertions(+), 329 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/FieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/FieldData.java index fce29c09ba40e..1d7e10bfcffbe 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/FieldData.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/FieldData.java @@ -22,15 +22,14 @@ import org.apache.lucene.index.BinaryDocValues; import org.apache.lucene.index.DocValues; import org.apache.lucene.index.NumericDocValues; -import org.apache.lucene.index.RandomAccessOrds; import org.apache.lucene.index.SortedNumericDocValues; import org.apache.lucene.index.SortedSetDocValues; -import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.NumericUtils; import org.elasticsearch.common.geo.GeoPoint; +import java.io.IOException; import java.util.ArrayList; import java.util.List; @@ -43,8 +42,8 @@ public enum FieldData { /** * Return a {@link SortedBinaryDocValues} that doesn't contain any value. */ - public static SortedBinaryDocValues emptySortedBinary(int maxDoc) { - return singleton(DocValues.emptyBinary(), new Bits.MatchNoBits(maxDoc)); + public static SortedBinaryDocValues emptySortedBinary() { + return singleton(DocValues.emptyBinary()); } /** @@ -53,8 +52,13 @@ public static SortedBinaryDocValues emptySortedBinary(int maxDoc) { public static NumericDoubleValues emptyNumericDouble() { return new NumericDoubleValues() { @Override - public double get(int docID) { - return 0; + public boolean advanceExact(int doc) throws IOException { + return false; + } + + @Override + public double doubleValue() throws IOException { + throw new UnsupportedOperationException(); } }; @@ -63,16 +67,20 @@ public double get(int docID) { /** * Return a {@link SortedNumericDoubleValues} that doesn't contain any value. */ - public static SortedNumericDoubleValues emptySortedNumericDoubles(int maxDoc) { - return singleton(emptyNumericDouble(), new Bits.MatchNoBits(maxDoc)); + public static SortedNumericDoubleValues emptySortedNumericDoubles() { + return singleton(emptyNumericDouble()); } public static GeoPointValues emptyGeoPoint() { - final GeoPoint point = new GeoPoint(); return new GeoPointValues() { @Override - public GeoPoint get(int docID) { - return point; + public boolean advanceExact(int doc) throws IOException { + return false; + } + + @Override + public GeoPoint geoPointValue() { + throw new UnsupportedOperationException(); } }; } @@ -80,62 +88,71 @@ public GeoPoint get(int docID) { /** * Return a {@link SortedNumericDoubleValues} that doesn't contain any value. */ - public static MultiGeoPointValues emptyMultiGeoPoints(int maxDoc) { - return singleton(emptyGeoPoint(), new Bits.MatchNoBits(maxDoc)); + public static MultiGeoPointValues emptyMultiGeoPoints() { + return singleton(emptyGeoPoint()); } /** * Returns a {@link Bits} representing all documents from dv that have a value. */ public static Bits docsWithValue(final SortedBinaryDocValues dv, final int maxDoc) { - return new Bits() { - @Override - public boolean get(int index) { - dv.setDocument(index); - return dv.count() != 0; - } + return new Bits() { + @Override + public boolean get(int index) { + try { + return dv.advanceExact(index); + } catch (IOException e) { + throw new RuntimeException(e); + } + } - @Override - public int length() { - return maxDoc; - } - }; + @Override + public int length() { + return maxDoc; + } + }; } /** * Returns a Bits representing all documents from dv that have a value. */ public static Bits docsWithValue(final MultiGeoPointValues dv, final int maxDoc) { - return new Bits() { - @Override - public boolean get(int index) { - dv.setDocument(index); - return dv.count() != 0; - } + return new Bits() { + @Override + public boolean get(int index) { + try { + return dv.advanceExact(index); + } catch (IOException e) { + throw new RuntimeException(e); + } + } - @Override - public int length() { - return maxDoc; - } - }; + @Override + public int length() { + return maxDoc; + } + }; } /** * Returns a Bits representing all documents from dv that have a value. */ public static Bits docsWithValue(final SortedNumericDoubleValues dv, final int maxDoc) { - return new Bits() { - @Override - public boolean get(int index) { - dv.setDocument(index); - return dv.count() != 0; - } + return new Bits() { + @Override + public boolean get(int index) { + try { + return dv.advanceExact(index); + } catch (IOException e) { + throw new RuntimeException(e); + } + } - @Override - public int length() { - return maxDoc; - } - }; + @Override + public int length() { + return maxDoc; + } + }; } /** @@ -152,8 +169,7 @@ public static SortedNumericDocValues toSortableLongBits(SortedNumericDoubleValue } else { longBits = new SortableLongBitsNumericDocValues(singleton); } - final Bits docsWithField = unwrapSingletonBits(values); - return DocValues.singleton(longBits, docsWithField); + return DocValues.singleton(longBits); } else { if (values instanceof SortableLongBitsToSortedNumericDoubleValues) { return ((SortableLongBitsToSortedNumericDoubleValues) values).getLongValues(); @@ -177,8 +193,7 @@ public static SortedNumericDoubleValues sortableLongBitsToDoubles(SortedNumericD } else { doubles = new SortableLongBitsToNumericDoubleValues(singleton); } - final Bits docsWithField = DocValues.unwrapSingletonBits(values); - return singleton(doubles, docsWithField); + return singleton(doubles); } else { if (values instanceof SortableLongBitsSortedNumericDocValues) { return ((SortableLongBitsSortedNumericDocValues) values).getDoubleValues(); @@ -194,8 +209,7 @@ public static SortedNumericDoubleValues sortableLongBitsToDoubles(SortedNumericD public static SortedNumericDoubleValues castToDouble(final SortedNumericDocValues values) { final NumericDocValues singleton = DocValues.unwrapSingleton(values); if (singleton != null) { - final Bits docsWithField = DocValues.unwrapSingletonBits(values); - return singleton(new DoubleCastedValues(singleton), docsWithField); + return singleton(new DoubleCastedValues(singleton)); } else { return new SortedDoubleCastedValues(values); } @@ -207,8 +221,7 @@ public static SortedNumericDoubleValues castToDouble(final SortedNumericDocValue public static SortedNumericDocValues castToLong(final SortedNumericDoubleValues values) { final NumericDoubleValues singleton = unwrapSingleton(values); if (singleton != null) { - final Bits docsWithField = unwrapSingletonBits(values); - return DocValues.singleton(new LongCastedValues(singleton), docsWithField); + return DocValues.singleton(new LongCastedValues(singleton)); } else { return new SortedLongCastedValues(values); } @@ -217,8 +230,8 @@ public static SortedNumericDocValues castToLong(final SortedNumericDoubleValues /** * Returns a multi-valued view over the provided {@link NumericDoubleValues}. */ - public static SortedNumericDoubleValues singleton(NumericDoubleValues values, Bits docsWithField) { - return new SingletonSortedNumericDoubleValues(values, docsWithField); + public static SortedNumericDoubleValues singleton(NumericDoubleValues values) { + return new SingletonSortedNumericDoubleValues(values); } /** @@ -234,24 +247,11 @@ public static NumericDoubleValues unwrapSingleton(SortedNumericDoubleValues valu return null; } - /** - * Returns the documents with a value for the {@link SortedNumericDoubleValues}, - * if it was previously wrapped with {@link #singleton(NumericDoubleValues, Bits)}, - * or null. - */ - public static Bits unwrapSingletonBits(SortedNumericDoubleValues dv) { - if (dv instanceof SingletonSortedNumericDoubleValues) { - return ((SingletonSortedNumericDoubleValues)dv).getDocsWithField(); - } else { - return null; - } - } - /** * Returns a multi-valued view over the provided {@link GeoPointValues}. */ - public static MultiGeoPointValues singleton(GeoPointValues values, Bits docsWithField) { - return new SingletonMultiGeoPointValues(values, docsWithField); + public static MultiGeoPointValues singleton(GeoPointValues values) { + return new SingletonMultiGeoPointValues(values); } /** @@ -267,23 +267,11 @@ public static GeoPointValues unwrapSingleton(MultiGeoPointValues values) { return null; } - /** - * Returns the documents with a value for the {@link MultiGeoPointValues}, - * if it was previously wrapped with {@link #singleton(GeoPointValues, Bits)}, - * or null. - */ - public static Bits unwrapSingletonBits(MultiGeoPointValues values) { - if (values instanceof SingletonMultiGeoPointValues) { - return ((SingletonMultiGeoPointValues) values).getDocsWithField(); - } - return null; - } - /** * Returns a multi-valued view over the provided {@link BinaryDocValues}. */ - public static SortedBinaryDocValues singleton(BinaryDocValues values, Bits docsWithField) { - return new SingletonSortedBinaryDocValues(values, docsWithField); + public static SortedBinaryDocValues singleton(BinaryDocValues values) { + return new SingletonSortedBinaryDocValues(values); } /** @@ -299,18 +287,6 @@ public static BinaryDocValues unwrapSingleton(SortedBinaryDocValues values) { return null; } - /** - * Returns the documents with a value for the {@link SortedBinaryDocValues}, - * if it was previously wrapped with {@link #singleton(BinaryDocValues, Bits)}, - * or null. - */ - public static Bits unwrapSingletonBits(SortedBinaryDocValues values) { - if (values instanceof SingletonSortedBinaryDocValues) { - return ((SingletonSortedBinaryDocValues) values).getDocsWithField(); - } - return null; - } - /** * Returns whether the provided values *might* be multi-valued. There is no * guarantee that this method will return false in the single-valued case. @@ -359,10 +335,13 @@ public static boolean isMultiValued(MultiGeoPointValues values) { public static SortedBinaryDocValues toString(final SortedNumericDocValues values) { return toString(new ToStringValues() { @Override - public void get(int docID, List list) { - values.setDocument(docID); - for (int i = 0, count = values.count(); i < count; ++i) { - list.add(Long.toString(values.valueAt(i))); + public boolean advanceExact(int doc) throws IOException { + return values.advanceExact(doc); + } + @Override + public void get(List list) throws IOException { + for (int i = 0, count = values.docValueCount(); i < count; ++i) { + list.add(Long.toString(values.nextValue())); } } }); @@ -376,10 +355,13 @@ public void get(int docID, List list) { public static SortedBinaryDocValues toString(final SortedNumericDoubleValues values) { return toString(new ToStringValues() { @Override - public void get(int docID, List list) { - values.setDocument(docID); - for (int i = 0, count = values.count(); i < count; ++i) { - list.add(Double.toString(values.valueAt(i))); + public boolean advanceExact(int doc) throws IOException { + return values.advanceExact(doc); + } + @Override + public void get(List list) throws IOException { + for (int i = 0, count = values.docValueCount(); i < count; ++i) { + list.add(Double.toString(values.nextValue())); } } }); @@ -390,23 +372,37 @@ public void get(int docID, List list) { * typically used for scripts or for the `map` execution mode of terms aggs. * NOTE: this is slow! */ - public static SortedBinaryDocValues toString(final RandomAccessOrds values) { + public static SortedBinaryDocValues toString(final SortedSetDocValues values) { return new SortedBinaryDocValues() { + private int count = 0; @Override - public BytesRef valueAt(int index) { - return values.lookupOrd(values.ordAt(index)); + public boolean advanceExact(int doc) throws IOException { + if (values.advanceExact(doc) == false) { + return false; + } + for (int i = 0; ; ++i) { + if (values.nextOrd() == SortedSetDocValues.NO_MORE_ORDS) { + count = i; + break; + } + } + // reset the iterator on the current doc + boolean advanced = values.advanceExact(doc); + assert advanced; + return true; } @Override - public void setDocument(int docId) { - values.setDocument(docId); + public int docValueCount() { + return count; } - + @Override - public int count() { - return values.cardinality(); + public BytesRef nextValue() throws IOException { + return values.lookupOrd(values.nextOrd()); } + }; } @@ -418,78 +414,30 @@ public int count() { public static SortedBinaryDocValues toString(final MultiGeoPointValues values) { return toString(new ToStringValues() { @Override - public void get(int docID, List list) { - values.setDocument(docID); - for (int i = 0, count = values.count(); i < count; ++i) { - list.add(values.valueAt(i).toString()); + public boolean advanceExact(int doc) throws IOException { + return values.advanceExact(doc); + } + @Override + public void get(List list) { + for (int i = 0, count = values.docValueCount(); i < count; ++i) { + list.add(values.nextValue().toString()); } } }); } - /** - * If dv is an instance of {@link RandomAccessOrds}, then return - * it, otherwise wrap it into a slow wrapper that implements random access. - */ - public static RandomAccessOrds maybeSlowRandomAccessOrds(final SortedSetDocValues dv) { - if (dv instanceof RandomAccessOrds) { - return (RandomAccessOrds) dv; - } else { - assert DocValues.unwrapSingleton(dv) == null : "this method expect singleton to return random-access ords"; - return new RandomAccessOrds() { - - int cardinality; - long[] ords = new long[0]; - int ord; - - @Override - public void setDocument(int docID) { - cardinality = 0; - dv.setDocument(docID); - for (long ord = dv.nextOrd(); ord != SortedSetDocValues.NO_MORE_ORDS; ord = dv.nextOrd()) { - ords = ArrayUtil.grow(ords, cardinality + 1); - ords[cardinality++] = ord; - } - ord = 0; - } - - @Override - public long nextOrd() { - return ords[ord++]; - } - - @Override - public BytesRef lookupOrd(long ord) { - return dv.lookupOrd(ord); - } - - @Override - public long getValueCount() { - return dv.getValueCount(); - } - - @Override - public long ordAt(int index) { - return ords[index]; - } - - @Override - public int cardinality() { - return cardinality; - } - }; - } - } - private static SortedBinaryDocValues toString(final ToStringValues toStringValues) { return new SortingBinaryDocValues() { final List list = new ArrayList<>(); @Override - public void setDocument(int docID) { + public boolean advanceExact(int docID) throws IOException { + if (toStringValues.advanceExact(docID) == false) { + return false; + } list.clear(); - toStringValues.get(docID, list); + toStringValues.get(list); count = list.size(); grow(); for (int i = 0; i < count; ++i) { @@ -497,6 +445,7 @@ public void setDocument(int docID) { values[i].copyChars(s); } sort(); + return true; } }; @@ -504,7 +453,14 @@ public void setDocument(int docID) { private interface ToStringValues { - void get(int docID, List values); + /** + * Advance this instance to the given document id + * @return true if there is a value for this document + */ + public abstract boolean advanceExact(int doc) throws IOException; + + /** Fill the list of charsquences with the list of values for the current document. */ + void get(List values) throws IOException; } @@ -517,8 +473,13 @@ private static class DoubleCastedValues extends NumericDoubleValues { } @Override - public double get(int docID) { - return values.get(docID); + public double doubleValue() throws IOException { + return values.longValue(); + } + + @Override + public boolean advanceExact(int doc) throws IOException { + return values.advanceExact(doc); } } @@ -532,18 +493,18 @@ private static class SortedDoubleCastedValues extends SortedNumericDoubleValues } @Override - public double valueAt(int index) { - return values.valueAt(index); + public boolean advanceExact(int target) throws IOException { + return values.advanceExact(target); } @Override - public void setDocument(int doc) { - values.setDocument(doc); + public double nextValue() throws IOException { + return values.nextValue(); } @Override - public int count() { - return values.count(); + public int docValueCount() { + return values.docValueCount(); } } @@ -557,10 +518,34 @@ private static class LongCastedValues extends NumericDocValues { } @Override - public long get(int docID) { - return (long) values.get(docID); + public boolean advanceExact(int target) throws IOException { + return values.advanceExact(target); + } + + @Override + public long longValue() throws IOException { + return (long) values.doubleValue(); + } + + @Override + public int docID() { + throw new UnsupportedOperationException(); + } + + @Override + public int nextDoc() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public int advance(int target) throws IOException { + throw new UnsupportedOperationException(); } + @Override + public long cost() { + throw new UnsupportedOperationException(); + } } private static class SortedLongCastedValues extends SortedNumericDocValues { @@ -572,18 +557,38 @@ private static class SortedLongCastedValues extends SortedNumericDocValues { } @Override - public long valueAt(int index) { - return (long) values.valueAt(index); + public boolean advanceExact(int target) throws IOException { + return values.advanceExact(target); + } + + @Override + public int docValueCount() { + return values.docValueCount(); + } + + @Override + public long nextValue() throws IOException { + return (long) values.nextValue(); + } + + @Override + public int docID() { + throw new UnsupportedOperationException(); + } + + @Override + public int nextDoc() throws IOException { + throw new UnsupportedOperationException(); } @Override - public void setDocument(int doc) { - values.setDocument(doc); + public int advance(int target) throws IOException { + throw new UnsupportedOperationException(); } @Override - public int count() { - return values.count(); + public long cost() { + throw new UnsupportedOperationException(); } } diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/GeoPointValues.java b/core/src/main/java/org/elasticsearch/index/fielddata/GeoPointValues.java index 8e8c8e67c9b42..8a679f18cfd7a 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/GeoPointValues.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/GeoPointValues.java @@ -21,17 +21,23 @@ import org.elasticsearch.common.geo.GeoPoint; +import java.io.IOException; + /** * Per-document geo-point values. */ public abstract class GeoPointValues { /** - * Get the {@link GeoPoint} associated with docID. + * Advance this instance to the given document id + * @return true if there is a value for this document + */ + public abstract boolean advanceExact(int doc) throws IOException; + + /** + * Get the {@link GeoPoint} associated with the current document. * The returned {@link GeoPoint} might be reused across calls. - * If the given docID does not have a value then the returned - * geo point mught have both latitude and longitude set to 0. */ - public abstract GeoPoint get(int docID); + public abstract GeoPoint geoPointValue(); } diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/MultiGeoPointValues.java b/core/src/main/java/org/elasticsearch/index/fielddata/MultiGeoPointValues.java index 6fa9c799dd71f..0e2bfcecdd157 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/MultiGeoPointValues.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/MultiGeoPointValues.java @@ -20,6 +20,8 @@ import org.elasticsearch.common.geo.GeoPoint; +import java.io.IOException; + /** * A stateful lightweight per document set of {@link GeoPoint} values. * To iterate over values in a document use the following pattern: @@ -44,28 +46,24 @@ protected MultiGeoPointValues() { } /** - * Sets iteration to the specified docID. - * @param docId document ID - * - * @see #valueAt(int) - * @see #count() + * Advance this instance to the given document id + * @return true if there is a value for this document */ - public abstract void setDocument(int docId); + public abstract boolean advanceExact(int doc) throws IOException; /** * Return the number of geo points the current document has. */ - public abstract int count(); + public abstract int docValueCount(); /** - * Return the i-th value associated with the current document. - * Behavior is undefined when i is undefined or greater than - * or equal to {@link #count()}. + * Return the next value associated with the current document. This must not be + * called more than {@link #docValueCount()} times. * * Note: the returned {@link GeoPoint} might be shared across invocations. * * @return the next value for the current docID set to {@link #setDocument(int)}. */ - public abstract GeoPoint valueAt(int i); + public abstract GeoPoint nextValue(); } diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/NumericDoubleValues.java b/core/src/main/java/org/elasticsearch/index/fielddata/NumericDoubleValues.java index 2cbbb0064f4c2..dd84ee14b89c6 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/NumericDoubleValues.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/NumericDoubleValues.java @@ -20,32 +20,47 @@ package org.elasticsearch.index.fielddata; import org.apache.lucene.index.NumericDocValues; +import org.apache.lucene.search.DoubleValues; + +import java.io.IOException; /** * A per-document numeric value. */ -public abstract class NumericDoubleValues { +public abstract class NumericDoubleValues extends DoubleValues { /** Sole constructor. (For invocation by subclass * constructors, typically implicit.) */ protected NumericDoubleValues() {} - - /** - * Returns the numeric value for the specified document ID. This must return - * 0d if the given doc ID has no value. - * @param docID document ID to lookup - * @return numeric value - */ - public abstract double get(int docID); // TODO: this interaction with sort comparators is really ugly... /** Returns numeric docvalues view of raw double bits */ public NumericDocValues getRawDoubleValues() { return new NumericDocValues() { - @Override - public long get(int docID) { - return Double.doubleToRawLongBits(NumericDoubleValues.this.get(docID)); - } + @Override + public boolean advanceExact(int target) throws IOException { + return NumericDoubleValues.this.advanceExact(target); + } + @Override + public long longValue() throws IOException { + return Double.doubleToRawLongBits(NumericDoubleValues.this.doubleValue()); + } + @Override + public int docID() { + throw new UnsupportedOperationException(); + } + @Override + public int nextDoc() throws IOException { + throw new UnsupportedOperationException(); + } + @Override + public int advance(int target) throws IOException { + throw new UnsupportedOperationException(); + } + @Override + public long cost() { + throw new UnsupportedOperationException(); + } }; } @@ -53,10 +68,30 @@ public long get(int docID) { /** Returns numeric docvalues view of raw float bits */ public NumericDocValues getRawFloatValues() { return new NumericDocValues() { - @Override - public long get(int docID) { - return Float.floatToRawIntBits((float)NumericDoubleValues.this.get(docID)); - } + @Override + public boolean advanceExact(int target) throws IOException { + return NumericDoubleValues.this.advanceExact(target); + } + @Override + public long longValue() throws IOException { + return Float.floatToRawIntBits((float)NumericDoubleValues.this.doubleValue()); + } + @Override + public int docID() { + throw new UnsupportedOperationException(); + } + @Override + public int nextDoc() throws IOException { + throw new UnsupportedOperationException(); + } + @Override + public int advance(int target) throws IOException { + throw new UnsupportedOperationException(); + } + @Override + public long cost() { + throw new UnsupportedOperationException(); + } }; } } diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/SingletonMultiGeoPointValues.java b/core/src/main/java/org/elasticsearch/index/fielddata/SingletonMultiGeoPointValues.java index 883444fabc734..bae522e7b5081 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/SingletonMultiGeoPointValues.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/SingletonMultiGeoPointValues.java @@ -19,48 +19,34 @@ package org.elasticsearch.index.fielddata; -import org.apache.lucene.util.Bits; import org.elasticsearch.common.geo.GeoPoint; +import java.io.IOException; + final class SingletonMultiGeoPointValues extends MultiGeoPointValues { private final GeoPointValues in; - private final Bits docsWithField; - private GeoPoint value; - private int count; - SingletonMultiGeoPointValues(GeoPointValues in, Bits docsWithField) { + SingletonMultiGeoPointValues(GeoPointValues in) { this.in = in; - this.docsWithField = docsWithField; } @Override - public void setDocument(int docID) { - value = in.get(docID); - if (value.lat() == Double.NaN && value.lon() == Double.NaN || (docsWithField != null && !docsWithField.get(docID))) { - count = 0; - } else { - count = 1; - } + public boolean advanceExact(int doc) throws IOException { + return in.advanceExact(doc); } @Override - public int count() { - return count; + public int docValueCount() { + return 1; } @Override - public GeoPoint valueAt(int index) { - assert index == 0; - return value; + public GeoPoint nextValue() { + return in.geoPointValue(); } - public GeoPointValues getGeoPointValues() { + GeoPointValues getGeoPointValues() { return in; } - - public Bits getDocsWithField() { - return docsWithField; - } - } diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/SingletonSortedBinaryDocValues.java b/core/src/main/java/org/elasticsearch/index/fielddata/SingletonSortedBinaryDocValues.java index 1a35f056f9f19..01f110a7b381e 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/SingletonSortedBinaryDocValues.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/SingletonSortedBinaryDocValues.java @@ -20,49 +20,35 @@ package org.elasticsearch.index.fielddata; import org.apache.lucene.index.BinaryDocValues; -import org.apache.lucene.util.Bits; -import org.apache.lucene.util.Bits.MatchAllBits; import org.apache.lucene.util.BytesRef; +import java.io.IOException; + final class SingletonSortedBinaryDocValues extends SortedBinaryDocValues { private final BinaryDocValues in; - private final Bits docsWithField; - private BytesRef value; - private int count; - SingletonSortedBinaryDocValues(BinaryDocValues in, Bits docsWithField) { + SingletonSortedBinaryDocValues(BinaryDocValues in) { this.in = in; - this.docsWithField = docsWithField instanceof MatchAllBits ? null : docsWithField; } @Override - public void setDocument(int docID) { - value = in.get(docID); - if (value.length == 0 && docsWithField != null && !docsWithField.get(docID)) { - count = 0; - } else { - count = 1; - } + public boolean advanceExact(int doc) throws IOException { + return in.advanceExact(doc); } @Override - public int count() { - return count; + public int docValueCount() { + return 1; } @Override - public BytesRef valueAt(int index) { - assert index == 0; - return value; + public BytesRef nextValue() throws IOException { + return in.binaryValue(); } public BinaryDocValues getBinaryDocValues() { return in; } - public Bits getDocsWithField() { - return docsWithField; - } - } diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/SingletonSortedNumericDoubleValues.java b/core/src/main/java/org/elasticsearch/index/fielddata/SingletonSortedNumericDoubleValues.java index 4207ac73a1a0c..af0768ac2c354 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/SingletonSortedNumericDoubleValues.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/SingletonSortedNumericDoubleValues.java @@ -19,8 +19,7 @@ package org.elasticsearch.index.fielddata; -import org.apache.lucene.util.Bits; -import org.apache.lucene.util.Bits.MatchAllBits; +import java.io.IOException; /** * Exposes multi-valued view over a single-valued instance. @@ -29,43 +28,30 @@ * that works for single or multi-valued types. */ final class SingletonSortedNumericDoubleValues extends SortedNumericDoubleValues { - private final NumericDoubleValues in; - private final Bits docsWithField; - private double value; - private int count; + private final NumericDoubleValues in; - SingletonSortedNumericDoubleValues(NumericDoubleValues in, Bits docsWithField) { - this.in = in; - this.docsWithField = docsWithField instanceof MatchAllBits ? null : docsWithField; - } + SingletonSortedNumericDoubleValues(NumericDoubleValues in) { + this.in = in; + } - /** Return the wrapped {@link NumericDoubleValues} */ - public NumericDoubleValues getNumericDoubleValues() { - return in; - } + /** Return the wrapped {@link NumericDoubleValues} */ + public NumericDoubleValues getNumericDoubleValues() { + return in; + } - /** Return the wrapped {@link Bits} */ - public Bits getDocsWithField() { - return docsWithField; - } + @Override + public boolean advanceExact(int target) throws IOException { + return in.advanceExact(target); + } - @Override - public void setDocument(int doc) { - value = in.get(doc); - if (docsWithField != null && value == 0 && docsWithField.get(doc) == false) { - count = 0; - } else { - count = 1; + @Override + public int docValueCount() { + return 1; } - } - @Override - public double valueAt(int index) { - return value; - } + @Override + public double nextValue() throws IOException { + return in.doubleValue(); + } - @Override - public int count() { - return count; - } } diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/SortedBinaryDocValues.java b/core/src/main/java/org/elasticsearch/index/fielddata/SortedBinaryDocValues.java index b3c51141e20f9..119fda1bda8f6 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/SortedBinaryDocValues.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/SortedBinaryDocValues.java @@ -21,28 +21,35 @@ import org.apache.lucene.util.BytesRef; +import java.io.IOException; + /** * A list of per-document binary values, sorted * according to {@link BytesRef#compareTo(BytesRef)}. * There might be dups however. */ +// TODO: Should it expose a count (current approach) or return null when there are no more values? public abstract class SortedBinaryDocValues { /** - * Positions to the specified document + * Advance this instance to the given document id + * @return true if there is a value for this document */ - public abstract void setDocument(int docId); + public abstract boolean advanceExact(int doc) throws IOException; - /** - * Return the number of values of the current document. + /** + * Retrieves the number of values for the current document. This must always + * be greater than zero. + * It is illegal to call this method after {@link #advanceExact(int)} + * returned {@code false}. */ - public abstract int count(); + public abstract int docValueCount(); - /** - * Retrieve the value for the current document at the specified index. - * An index ranges from {@code 0} to {@code count()-1}. + /** + * Iterates to the next value in the current document. Do not call this more than + * {@link #docValueCount} times for the document. * Note that the returned {@link BytesRef} might be reused across invocations. */ - public abstract BytesRef valueAt(int index); + public abstract BytesRef nextValue() throws IOException; } diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/SortedNumericDoubleValues.java b/core/src/main/java/org/elasticsearch/index/fielddata/SortedNumericDoubleValues.java index 23cdfa7c0d15f..e036e9dd68896 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/SortedNumericDoubleValues.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/SortedNumericDoubleValues.java @@ -21,6 +21,8 @@ import org.apache.lucene.index.SortedNumericDocValues; +import java.io.IOException; + /** * Clone of {@link SortedNumericDocValues} for double values. */ @@ -30,21 +32,26 @@ public abstract class SortedNumericDoubleValues { * constructors, typically implicit.) */ protected SortedNumericDoubleValues() {} - /** - * Positions to the specified document - */ - public abstract void setDocument(int doc); - - /** - * Retrieve the value for the current document at the specified index. - * An index ranges from {@code 0} to {@code count()-1}. + /** Advance the iterator to exactly {@code target} and return whether + * {@code target} has a value. + * {@code target} must be greater than or equal to the current + * {@link #docID() doc ID} and must be a valid doc ID, ie. ≥ 0 and + * < {@code maxDoc}. + * After this method returns, {@link #docID()} retuns {@code target}. */ + public abstract boolean advanceExact(int target) throws IOException; + + /** + * Iterates to the next value in the current document. Do not call this more than + * {@link #docValueCount} times for the document. */ - public abstract double valueAt(int index); - - /** - * Retrieves the count of values for the current document. - * This may be zero if a document has no values. + public abstract double nextValue() throws IOException; + + /** + * Retrieves the number of values for the current document. This must always + * be greater than zero. + * It is illegal to call this method after {@link #advanceExact(int)} + * returned {@code false}. */ - public abstract int count(); + public abstract int docValueCount(); } diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/SortingBinaryDocValues.java b/core/src/main/java/org/elasticsearch/index/fielddata/SortingBinaryDocValues.java index c35d59fc692d4..99f0d477254fe 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/SortingBinaryDocValues.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/SortingBinaryDocValues.java @@ -33,6 +33,7 @@ */ public abstract class SortingBinaryDocValues extends SortedBinaryDocValues { + private int index; protected int count; protected BytesRefBuilder[] values; private final Sorter sorter; @@ -73,15 +74,17 @@ protected final void grow() { */ protected final void sort() { sorter.sort(0, count); + index = 0; } @Override - public final int count() { + public int docValueCount() { return count; } @Override - public final BytesRef valueAt(int index) { - return values[index].get(); + public final BytesRef nextValue() { + assert index < count; + return values[index++].get(); } } From d0da10317a4a2816c5330109f35c418d467cd202 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Wed, 5 Apr 2017 09:45:19 +0200 Subject: [PATCH 03/87] AnalyzingQueryParser has been replaced by QueryParser. --- .../classic/MapperQueryParser.java | 35 +++++-------------- 1 file changed, 9 insertions(+), 26 deletions(-) diff --git a/core/src/main/java/org/apache/lucene/queryparser/classic/MapperQueryParser.java b/core/src/main/java/org/apache/lucene/queryparser/classic/MapperQueryParser.java index 22ecf37c3a66e..a61e235019006 100644 --- a/core/src/main/java/org/apache/lucene/queryparser/classic/MapperQueryParser.java +++ b/core/src/main/java/org/apache/lucene/queryparser/classic/MapperQueryParser.java @@ -24,9 +24,7 @@ import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute; import org.apache.lucene.index.Term; -import org.apache.lucene.queryparser.analyzing.AnalyzingQueryParser; import org.apache.lucene.search.BooleanClause; -import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.BoostQuery; import org.apache.lucene.search.DisjunctionMaxQuery; import org.apache.lucene.search.FuzzyQuery; @@ -56,7 +54,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Objects; import static java.util.Collections.unmodifiableMap; import static org.elasticsearch.common.lucene.search.Queries.fixNegativeQueryIfNeeded; @@ -68,7 +65,7 @@ * Also breaks fields with [type].[name] into a boolean query that must include the type * as well as the query on the name. */ -public class MapperQueryParser extends AnalyzingQueryParser { +public class MapperQueryParser extends QueryParser { public static final Map FIELD_QUERY_EXTENSIONS; @@ -104,7 +101,6 @@ public void reset(QueryParserSettings settings) { setAutoGeneratePhraseQueries(settings.autoGeneratePhraseQueries()); setMaxDeterminizedStates(settings.maxDeterminizedStates()); setAllowLeadingWildcard(settings.allowLeadingWildcard()); - setLowercaseExpandedTerms(false); setPhraseSlop(settings.phraseSlop()); setDefaultOperator(settings.defaultOperator()); setFuzzyPrefixLength(settings.fuzzyPrefixLength()); @@ -173,7 +169,7 @@ public Query getFieldQuery(String field, String queryText, boolean quoted) throw } } if (clauses.isEmpty()) return null; // happens for stopwords - return getBooleanQueryCoordDisabled(clauses); + return getBooleanQuery(clauses); } } else { return getFieldQuerySingle(field, queryText, quoted); @@ -275,7 +271,7 @@ protected Query getFieldQuery(String field, String queryText, int slop) throws P } } if (clauses.isEmpty()) return null; // happens for stopwords - return getBooleanQueryCoordDisabled(clauses); + return getBooleanQuery(clauses); } } else { return super.getFieldQuery(field, queryText, slop); @@ -326,7 +322,7 @@ protected Query getRangeQuery(String field, String part1, String part2, } } if (clauses.isEmpty()) return null; // happens for stopwords - return getBooleanQueryCoordDisabled(clauses); + return getBooleanQuery(clauses); } } @@ -384,7 +380,7 @@ protected Query getFuzzyQuery(String field, String termStr, String minSimilarity clauses.add(new BooleanClause(applyBoost(mField, q), BooleanClause.Occur.SHOULD)); } } - return getBooleanQueryCoordDisabled(clauses); + return getBooleanQuery(clauses); } } else { return getFuzzyQuerySingle(field, termStr, minSimilarity); @@ -448,7 +444,7 @@ protected Query getPrefixQuery(String field, String termStr) throws ParseExcepti } } if (clauses.isEmpty()) return null; // happens for stopwords - return getBooleanQueryCoordDisabled(clauses); + return getBooleanQuery(clauses); } } else { return getPrefixQuerySingle(field, termStr); @@ -557,7 +553,7 @@ private Query getPossiblyAnalyzedPrefixQuery(String field, String termStr) throw innerClauses.add(new BooleanClause(super.getPrefixQuery(field, token), BooleanClause.Occur.SHOULD)); } - posQuery = getBooleanQueryCoordDisabled(innerClauses); + posQuery = getBooleanQuery(innerClauses); } clauses.add(new BooleanClause(posQuery, getDefaultOperator() == Operator.AND ? BooleanClause.Occur.MUST : BooleanClause.Occur.SHOULD)); @@ -606,7 +602,7 @@ protected Query getWildcardQuery(String field, String termStr) throws ParseExcep } } if (clauses.isEmpty()) return null; // happens for stopwords - return getBooleanQueryCoordDisabled(clauses); + return getBooleanQuery(clauses); } } else { return getWildcardQuerySingle(field, termStr); @@ -670,7 +666,7 @@ protected Query getRegexpQuery(String field, String termStr) throws ParseExcepti } } if (clauses.isEmpty()) return null; // happens for stopwords - return getBooleanQueryCoordDisabled(clauses); + return getBooleanQuery(clauses); } } else { return getRegexpQuerySingle(field, termStr); @@ -707,19 +703,6 @@ private Query getRegexpQuerySingle(String field, String termStr) throws ParseExc } } - /** - * @deprecated review all use of this, don't rely on coord - */ - @Deprecated - protected Query getBooleanQueryCoordDisabled(List clauses) throws ParseException { - BooleanQuery.Builder builder = new BooleanQuery.Builder(); - builder.setDisableCoord(true); - for (BooleanClause clause : clauses) { - builder.add(clause); - } - return fixNegativeQueryIfNeeded(builder.build()); - } - @Override protected Query getBooleanQuery(List clauses) throws ParseException { From fef7a60402472216bc9e35740825994535a3f76b Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Wed, 5 Apr 2017 09:45:40 +0200 Subject: [PATCH 04/87] Query API changes. --- .../org/apache/lucene/queries/BlendedTermQuery.java | 13 ++++--------- .../lucene/queries/ExtendedCommonTermsQuery.java | 4 ++-- .../java/org/apache/lucene/queries/MinDocQuery.java | 4 ++-- 3 files changed, 8 insertions(+), 13 deletions(-) diff --git a/core/src/main/java/org/apache/lucene/queries/BlendedTermQuery.java b/core/src/main/java/org/apache/lucene/queries/BlendedTermQuery.java index 0b34a95710cfe..dbad7e0bf721b 100644 --- a/core/src/main/java/org/apache/lucene/queries/BlendedTermQuery.java +++ b/core/src/main/java/org/apache/lucene/queries/BlendedTermQuery.java @@ -296,16 +296,15 @@ public int hashCode() { return Objects.hash(classHash(), Arrays.hashCode(equalsTerms())); } - public static BlendedTermQuery booleanBlendedQuery(Term[] terms, final boolean disableCoord) { - return booleanBlendedQuery(terms, null, disableCoord); + public static BlendedTermQuery booleanBlendedQuery(Term[] terms) { + return booleanBlendedQuery(terms, null); } - public static BlendedTermQuery booleanBlendedQuery(Term[] terms, final float[] boosts, final boolean disableCoord) { + public static BlendedTermQuery booleanBlendedQuery(Term[] terms, final float[] boosts) { return new BlendedTermQuery(terms, boosts) { @Override protected Query topLevelQuery(Term[] terms, TermContext[] ctx, int[] docFreqs, int maxDoc) { BooleanQuery.Builder booleanQueryBuilder = new BooleanQuery.Builder(); - booleanQueryBuilder.setDisableCoord(disableCoord); for (int i = 0; i < terms.length; i++) { Query query = new TermQuery(terms[i], ctx[i]); if (boosts != null && boosts[i] != 1f) { @@ -318,14 +317,12 @@ protected Query topLevelQuery(Term[] terms, TermContext[] ctx, int[] docFreqs, i }; } - public static BlendedTermQuery commonTermsBlendedQuery(Term[] terms, final float[] boosts, final boolean disableCoord, final float maxTermFrequency) { + public static BlendedTermQuery commonTermsBlendedQuery(Term[] terms, final float[] boosts, final float maxTermFrequency) { return new BlendedTermQuery(terms, boosts) { @Override protected Query topLevelQuery(Term[] terms, TermContext[] ctx, int[] docFreqs, int maxDoc) { BooleanQuery.Builder highBuilder = new BooleanQuery.Builder(); - highBuilder.setDisableCoord(disableCoord); BooleanQuery.Builder lowBuilder = new BooleanQuery.Builder(); - lowBuilder.setDisableCoord(disableCoord); for (int i = 0; i < terms.length; i++) { Query query = new TermQuery(terms[i], ctx[i]); if (boosts != null && boosts[i] != 1f) { @@ -343,7 +340,6 @@ protected Query topLevelQuery(Term[] terms, TermContext[] ctx, int[] docFreqs, i BooleanQuery low = lowBuilder.build(); if (low.clauses().isEmpty()) { BooleanQuery.Builder queryBuilder = new BooleanQuery.Builder(); - queryBuilder.setDisableCoord(disableCoord); for (BooleanClause booleanClause : high) { queryBuilder.add(booleanClause.getQuery(), Occur.MUST); } @@ -352,7 +348,6 @@ protected Query topLevelQuery(Term[] terms, TermContext[] ctx, int[] docFreqs, i return low; } else { return new BooleanQuery.Builder() - .setDisableCoord(true) .add(high, BooleanClause.Occur.SHOULD) .add(low, BooleanClause.Occur.MUST) .build(); diff --git a/core/src/main/java/org/apache/lucene/queries/ExtendedCommonTermsQuery.java b/core/src/main/java/org/apache/lucene/queries/ExtendedCommonTermsQuery.java index 1889c6e759b11..4580de4cc4a00 100644 --- a/core/src/main/java/org/apache/lucene/queries/ExtendedCommonTermsQuery.java +++ b/core/src/main/java/org/apache/lucene/queries/ExtendedCommonTermsQuery.java @@ -35,8 +35,8 @@ public class ExtendedCommonTermsQuery extends CommonTermsQuery { private final MappedFieldType fieldType; - public ExtendedCommonTermsQuery(Occur highFreqOccur, Occur lowFreqOccur, float maxTermFrequency, boolean disableCoord, MappedFieldType fieldType) { - super(highFreqOccur, lowFreqOccur, maxTermFrequency, disableCoord); + public ExtendedCommonTermsQuery(Occur highFreqOccur, Occur lowFreqOccur, float maxTermFrequency, MappedFieldType fieldType) { + super(highFreqOccur, lowFreqOccur, maxTermFrequency); this.fieldType = fieldType; } diff --git a/core/src/main/java/org/apache/lucene/queries/MinDocQuery.java b/core/src/main/java/org/apache/lucene/queries/MinDocQuery.java index a8b7dc9299ff0..d4f9ab729736c 100644 --- a/core/src/main/java/org/apache/lucene/queries/MinDocQuery.java +++ b/core/src/main/java/org/apache/lucene/queries/MinDocQuery.java @@ -57,8 +57,8 @@ public boolean equals(Object obj) { } @Override - public Weight createWeight(IndexSearcher searcher, boolean needsScores) throws IOException { - return new ConstantScoreWeight(this) { + public Weight createWeight(IndexSearcher searcher, boolean needsScores, float boost) throws IOException { + return new ConstantScoreWeight(this, boost) { @Override public Scorer scorer(LeafReaderContext context) throws IOException { final int maxDoc = context.reader().maxDoc(); From 2439ca9a1c77aa9a1d764dc4908e849b81f03fac Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Wed, 5 Apr 2017 10:43:49 +0200 Subject: [PATCH 05/87] Changes to per-reader/per-segment caching. --- .../common/lucene/ShardCoreKeyMap.java | 26 ++++++++------- .../index/ElasticsearchDirectoryReader.java | 21 ++++++++---- .../common/lucene/uid/Versions.java | 32 +++++++++++------- .../index/cache/bitset/BitsetFilterCache.java | 22 ++++++++----- .../cache/IndicesFieldDataCache.java | 33 ++++++++++--------- 5 files changed, 82 insertions(+), 52 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/common/lucene/ShardCoreKeyMap.java b/core/src/main/java/org/elasticsearch/common/lucene/ShardCoreKeyMap.java index 38e7691f398f1..73255aef13330 100644 --- a/core/src/main/java/org/elasticsearch/common/lucene/ShardCoreKeyMap.java +++ b/core/src/main/java/org/elasticsearch/common/lucene/ShardCoreKeyMap.java @@ -19,8 +19,8 @@ package org.elasticsearch.common.lucene; +import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.LeafReader; -import org.apache.lucene.index.LeafReader.CoreClosedListener; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.index.shard.ShardUtils; @@ -46,8 +46,8 @@ */ public final class ShardCoreKeyMap { - private final Map coreKeyToShard; - private final Map> indexToCoreKey; + private final Map coreKeyToShard; + private final Map> indexToCoreKey; public ShardCoreKeyMap() { coreKeyToShard = new ConcurrentHashMap<>(); @@ -63,7 +63,11 @@ public void add(LeafReader reader) { if (shardId == null) { throw new IllegalArgumentException("Could not extract shard id from " + reader); } - final Object coreKey = reader.getCoreCacheKey(); + final IndexReader.CacheHelper cacheHelper = reader.getCoreCacheHelper(); + if (cacheHelper == null) { + throw new IllegalArgumentException("Reader " + reader + " does not support caching"); + } + final IndexReader.CacheKey coreKey = cacheHelper.getKey(); if (coreKeyToShard.containsKey(coreKey)) { // Do this check before entering the synchronized block in order to @@ -75,18 +79,18 @@ public void add(LeafReader reader) { final String index = shardId.getIndexName(); synchronized (this) { if (coreKeyToShard.containsKey(coreKey) == false) { - Set objects = indexToCoreKey.get(index); + Set objects = indexToCoreKey.get(index); if (objects == null) { objects = new HashSet<>(); indexToCoreKey.put(index, objects); } final boolean added = objects.add(coreKey); assert added; - CoreClosedListener listener = ownerCoreCacheKey -> { + IndexReader.ClosedListener listener = ownerCoreCacheKey -> { assert coreKey == ownerCoreCacheKey; synchronized (ShardCoreKeyMap.this) { coreKeyToShard.remove(ownerCoreCacheKey); - final Set coreKeys = indexToCoreKey.get(index); + final Set coreKeys = indexToCoreKey.get(index); final boolean removed = coreKeys.remove(coreKey); assert removed; if (coreKeys.isEmpty()) { @@ -96,7 +100,7 @@ public void add(LeafReader reader) { }; boolean addedListener = false; try { - reader.addCoreClosedListener(listener); + cacheHelper.addClosedListener(listener); addedListener = true; // Only add the core key to the map as a last operation so that @@ -131,7 +135,7 @@ public synchronized ShardId getShardId(Object coreKey) { * Get the set of core cache keys associated with the given index. */ public synchronized Set getCoreKeysForIndex(String index) { - final Set objects = indexToCoreKey.get(index); + final Set objects = indexToCoreKey.get(index); if (objects == null) { return Collections.emptySet(); } @@ -154,9 +158,9 @@ private synchronized boolean assertSize() { if (assertionsEnabled == false) { throw new AssertionError("only run this if assertions are enabled"); } - Collection> values = indexToCoreKey.values(); + Collection> values = indexToCoreKey.values(); int size = 0; - for (Set value : values) { + for (Set value : values) { size += value.size(); } return size == coreKeyToShard.size(); diff --git a/core/src/main/java/org/elasticsearch/common/lucene/index/ElasticsearchDirectoryReader.java b/core/src/main/java/org/elasticsearch/common/lucene/index/ElasticsearchDirectoryReader.java index b9caecea965c7..0f83e07874f8a 100644 --- a/core/src/main/java/org/elasticsearch/common/lucene/index/ElasticsearchDirectoryReader.java +++ b/core/src/main/java/org/elasticsearch/common/lucene/index/ElasticsearchDirectoryReader.java @@ -49,6 +49,12 @@ public ShardId shardId() { return this.shardId; } + @Override + public CacheHelper getReaderCacheHelper() { + // safe to delegate since this reader does not alter the index + return in.getReaderCacheHelper(); + } + @Override protected DirectoryReader doWrapDirectoryReader(DirectoryReader in) throws IOException { return new ElasticsearchDirectoryReader(in, wrapper, shardId); @@ -84,14 +90,17 @@ public LeafReader wrap(LeafReader reader) { * @throws IllegalArgumentException if the reader doesn't contain an {@link ElasticsearchDirectoryReader} in it's hierarchy */ @SuppressForbidden(reason = "This is the only sane way to add a ReaderClosedListener") - public static void addReaderCloseListener(DirectoryReader reader, IndexReader.ReaderClosedListener listener) { + public static void addReaderCloseListener(DirectoryReader reader, IndexReader.ClosedListener listener) { ElasticsearchDirectoryReader elasticsearchDirectoryReader = getElasticsearchDirectoryReader(reader); - if (elasticsearchDirectoryReader != null) { - assert reader.getCoreCacheKey() == elasticsearchDirectoryReader.getCoreCacheKey(); - elasticsearchDirectoryReader.addReaderClosedListener(listener); - return; + if (elasticsearchDirectoryReader == null) { + throw new IllegalArgumentException("Can't install close listener reader is not an ElasticsearchDirectoryReader/ElasticsearchLeafReader"); + } + IndexReader.CacheHelper cacheHelper = elasticsearchDirectoryReader.getReaderCacheHelper(); + if (cacheHelper == null) { + throw new IllegalArgumentException("Reader " + elasticsearchDirectoryReader + " does not support caching"); } - throw new IllegalArgumentException("Can't install close listener reader is not an ElasticsearchDirectoryReader/ElasticsearchLeafReader"); + assert cacheHelper.getKey() == reader.getReaderCacheHelper().getKey(); + cacheHelper.addClosedListener(listener); } /** diff --git a/core/src/main/java/org/elasticsearch/common/lucene/uid/Versions.java b/core/src/main/java/org/elasticsearch/common/lucene/uid/Versions.java index 6c5ffed0938e6..84110e3815607 100644 --- a/core/src/main/java/org/elasticsearch/common/lucene/uid/Versions.java +++ b/core/src/main/java/org/elasticsearch/common/lucene/uid/Versions.java @@ -22,7 +22,6 @@ import org.apache.lucene.index.Fields; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.LeafReader; -import org.apache.lucene.index.LeafReader.CoreClosedListener; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.NumericDocValues; import org.apache.lucene.index.PostingsEnum; @@ -61,12 +60,12 @@ public class Versions { public static final long MATCH_DELETED = -4L; // TODO: is there somewhere else we can store these? - static final ConcurrentMap> lookupStates = ConcurrentCollections.newConcurrentMapWithAggressiveConcurrency(); + static final ConcurrentMap> lookupStates = ConcurrentCollections.newConcurrentMapWithAggressiveConcurrency(); // Evict this reader from lookupStates once it's closed: - private static final CoreClosedListener removeLookupState = new CoreClosedListener() { + private static final IndexReader.ClosedListener removeLookupState = new IndexReader.ClosedListener() { @Override - public void onClose(Object key) { + public void onClose(IndexReader.CacheKey key) { CloseableThreadLocal ctl = lookupStates.remove(key); if (ctl != null) { ctl.close(); @@ -75,7 +74,12 @@ public void onClose(Object key) { }; private static PerThreadIDAndVersionLookup getLookupState(LeafReader reader) throws IOException { - Object key = reader.getCoreCacheKey(); + IndexReader.CacheHelper cacheHelper = reader.getCoreCacheHelper(); + if (cacheHelper == null) { + throw new IllegalArgumentException("Reader " + reader + " does not support caching"); + } + + IndexReader.CacheKey key = cacheHelper.getKey(); CloseableThreadLocal ctl = lookupStates.get(key); if (ctl == null) { // First time we are seeing this reader's core; make a @@ -85,7 +89,7 @@ private static PerThreadIDAndVersionLookup getLookupState(LeafReader reader) thr if (other == null) { // Our CTL won, we must remove it when the // core is closed: - reader.addCoreClosedListener(removeLookupState); + cacheHelper.addClosedListener(removeLookupState); } else { // Another thread beat us to it: just use // their CTL: @@ -200,10 +204,13 @@ public static long loadSeqNo(IndexReader reader, Term term) throws IOException { } if (docID != DocIdSetIterator.NO_MORE_DOCS) { - dvField.setDocument(docID); - assert dvField.count() == 1 : "expected only a single value for _seq_no but got " + - dvField.count(); - return dvField.valueAt(0); + if (dvField.advanceExact(docID) == false) { + throw new AssertionError("Document " + docID + " does not have a value for the " + SeqNoFieldMapper.NAME + " field"); + } + if (dvField.docValueCount() != 1) { + throw new AssertionError("Document " + docID + " has more than 1 value for the " + SeqNoFieldMapper.NAME + " field: " + dvField.docValueCount()); + } + return dvField.nextValue(); } } } @@ -256,7 +263,10 @@ public static long loadPrimaryTerm(IndexReader reader, Term term) throws IOExcep } if (docID != DocIdSetIterator.NO_MORE_DOCS) { - return dvField.get(docID); + if (dvField.advanceExact(docID) == false) { + throw new AssertionError("Document " + docID + " does not have a value for the " + SeqNoFieldMapper.PRIMARY_TERM_NAME + " field"); + } + return dvField.longValue(); } } } diff --git a/core/src/main/java/org/elasticsearch/index/cache/bitset/BitsetFilterCache.java b/core/src/main/java/org/elasticsearch/index/cache/bitset/BitsetFilterCache.java index 0e4c54e7a7d34..04d2ac47d187e 100644 --- a/core/src/main/java/org/elasticsearch/index/cache/bitset/BitsetFilterCache.java +++ b/core/src/main/java/org/elasticsearch/index/cache/bitset/BitsetFilterCache.java @@ -21,8 +21,8 @@ import org.apache.logging.log4j.message.ParameterizedMessage; import org.apache.logging.log4j.util.Supplier; +import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexReaderContext; -import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.ReaderUtil; import org.apache.lucene.search.IndexSearcher; @@ -71,13 +71,13 @@ * and require that it should always be around should use this cache, otherwise the * {@link org.elasticsearch.index.cache.query.QueryCache} should be used instead. */ -public final class BitsetFilterCache extends AbstractIndexComponent implements LeafReader.CoreClosedListener, RemovalListener>, Closeable { +public final class BitsetFilterCache extends AbstractIndexComponent implements IndexReader.ClosedListener, RemovalListener>, Closeable { public static final Setting INDEX_LOAD_RANDOM_ACCESS_FILTERS_EAGERLY_SETTING = Setting.boolSetting("index.load_fixed_bitset_filters_eagerly", true, Property.IndexScope); private final boolean loadRandomAccessFiltersEagerly; - private final Cache> loadedFilters; + private final Cache> loadedFilters; private final Listener listener; public BitsetFilterCache(IndexSettings indexSettings, Listener listener) { @@ -86,7 +86,7 @@ public BitsetFilterCache(IndexSettings indexSettings, Listener listener) { throw new IllegalArgumentException("listener must not be null"); } this.loadRandomAccessFiltersEagerly = this.indexSettings.getValue(INDEX_LOAD_RANDOM_ACCESS_FILTERS_EAGERLY_SETTING); - this.loadedFilters = CacheBuilder.>builder().removalListener(this).build(); + this.loadedFilters = CacheBuilder.>builder().removalListener(this).build(); this.listener = listener; } @@ -100,7 +100,7 @@ public BitSetProducer getBitSetProducer(Query query) { } @Override - public void onClose(Object ownerCoreCacheKey) { + public void onClose(IndexReader.CacheKey ownerCoreCacheKey) { loadedFilters.invalidate(ownerCoreCacheKey); } @@ -115,7 +115,11 @@ public void clear(String reason) { } private BitSet getAndLoadIfNotPresent(final Query query, final LeafReaderContext context) throws IOException, ExecutionException { - final Object coreCacheReader = context.reader().getCoreCacheKey(); + final IndexReader.CacheHelper cacheHelper = context.reader().getCoreCacheHelper(); + if (cacheHelper == null) { + throw new IllegalArgumentException("Reader " + context.reader() + " does not support caching"); + } + final IndexReader.CacheKey coreCacheReader = cacheHelper.getKey(); final ShardId shardId = ShardUtils.extractShardId(context.reader()); if (shardId != null // can't require it because of the percolator && indexSettings.getIndex().equals(shardId.getIndex()) == false) { @@ -124,7 +128,7 @@ private BitSet getAndLoadIfNotPresent(final Query query, final LeafReaderContext + " with cache of index " + indexSettings.getIndex()); } Cache filterToFbs = loadedFilters.computeIfAbsent(coreCacheReader, key -> { - context.reader().addCoreClosedListener(BitsetFilterCache.this); + cacheHelper.addClosedListener(BitsetFilterCache.this); return CacheBuilder.builder().build(); }); @@ -148,7 +152,7 @@ private BitSet getAndLoadIfNotPresent(final Query query, final LeafReaderContext } @Override - public void onRemoval(RemovalNotification> notification) { + public void onRemoval(RemovalNotification> notification) { if (notification.getKey() == null) { return; } @@ -272,7 +276,7 @@ public IndexWarmer.TerminationHandle warmReader(final IndexShard indexShard, fin } - Cache> getLoadedFilters() { + Cache> getLoadedFilters() { return loadedFilters; } diff --git a/core/src/main/java/org/elasticsearch/indices/fielddata/cache/IndicesFieldDataCache.java b/core/src/main/java/org/elasticsearch/indices/fielddata/cache/IndicesFieldDataCache.java index 860bdeb8ff651..fcd925c2585b2 100644 --- a/core/src/main/java/org/elasticsearch/indices/fielddata/cache/IndicesFieldDataCache.java +++ b/core/src/main/java/org/elasticsearch/indices/fielddata/cache/IndicesFieldDataCache.java @@ -23,7 +23,7 @@ import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.SegmentReader; +import org.apache.lucene.index.IndexReader.CacheKey; import org.apache.lucene.util.Accountable; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.cache.Cache; @@ -44,6 +44,7 @@ import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.index.shard.ShardUtils; +import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.function.ToLongBiFunction; @@ -107,7 +108,7 @@ public long applyAsLong(Key key, Accountable ramUsage) { /** * A specific cache instance for the relevant parameters of it (index, fieldNames, fieldType). */ - static class IndexFieldCache implements IndexFieldDataCache, SegmentReader.CoreClosedListener, IndexReader.ReaderClosedListener { + static class IndexFieldCache implements IndexFieldDataCache, IndexReader.ClosedListener { private final Logger logger; final Index index; final String fieldName; @@ -125,10 +126,14 @@ static class IndexFieldCache implements IndexFieldDataCache, SegmentReader.CoreC @Override public > FD load(final LeafReaderContext context, final IFD indexFieldData) throws Exception { final ShardId shardId = ShardUtils.extractShardId(context.reader()); - final Key key = new Key(this, context.reader().getCoreCacheKey(), shardId); + final IndexReader.CacheHelper cacheHelper = context.reader().getCoreCacheHelper(); + if (cacheHelper == null) { + throw new IllegalArgumentException("Reader " + context.reader() + " does not support caching"); + } + final Key key = new Key(this, cacheHelper.getKey(), shardId); //noinspection unchecked final Accountable accountable = cache.computeIfAbsent(key, k -> { - context.reader().addCoreClosedListener(IndexFieldCache.this); + cacheHelper.addClosedListener(IndexFieldCache.this); for (Listener listener : this.listeners) { k.listeners.add(listener); } @@ -149,7 +154,11 @@ public > FD load(fina @Override public > IFD load(final DirectoryReader indexReader, final IFD indexFieldData) throws Exception { final ShardId shardId = ShardUtils.extractShardId(indexReader); - final Key key = new Key(this, indexReader.getCoreCacheKey(), shardId); + final IndexReader.CacheHelper cacheHelper = indexReader.getReaderCacheHelper(); + if (cacheHelper == null) { + throw new IllegalArgumentException("Reader " + indexReader + " does not support caching"); + } + final Key key = new Key(this, cacheHelper.getKey(), shardId); //noinspection unchecked final Accountable accountable = cache.computeIfAbsent(key, k -> { ElasticsearchDirectoryReader.addReaderCloseListener(indexReader, IndexFieldCache.this); @@ -171,14 +180,8 @@ public > IFD l } @Override - public void onClose(Object coreKey) { - cache.invalidate(new Key(this, coreKey, null)); - // don't call cache.cleanUp here as it would have bad performance implications - } - - @Override - public void onClose(IndexReader reader) { - cache.invalidate(new Key(this, reader.getCoreCacheKey(), null)); + public void onClose(CacheKey key) throws IOException { + cache.invalidate(new Key(this, key, null)); // don't call cache.cleanUp here as it would have bad performance implications } @@ -211,12 +214,12 @@ public void clear(String fieldName) { public static class Key { public final IndexFieldCache indexCache; - public final Object readerKey; + public final IndexReader.CacheKey readerKey; public final ShardId shardId; public final List listeners = new ArrayList<>(); - Key(IndexFieldCache indexCache, Object readerKey, @Nullable ShardId shardId) { + Key(IndexFieldCache indexCache, IndexReader.CacheKey readerKey, @Nullable ShardId shardId) { this.indexCache = indexCache; this.readerKey = readerKey; this.shardId = shardId; From be8935240ab0d8fe32349eb2849cc3688727c6df Mon Sep 17 00:00:00 2001 From: Colin Goodheart-Smithe Date: Wed, 5 Apr 2017 12:45:36 +0100 Subject: [PATCH 06/87] Fixes compile errors for numeric aggregations --- .../bucket/geogrid/GeoHashGridAggregator.java | 29 ++++---- .../histogram/DateHistogramAggregator.java | 37 +++++----- .../bucket/histogram/HistogramAggregator.java | 37 +++++----- .../bucket/range/RangeAggregator.java | 11 +-- .../metrics/avg/AvgAggregator.java | 15 +++-- .../geobounds/GeoBoundsAggregator.java | 67 ++++++++++--------- .../geocentroid/GeoCentroidAggregator.java | 13 ++-- .../metrics/max/MaxAggregator.java | 10 +-- .../metrics/min/MinAggregator.java | 10 +-- .../hdr/AbstractHDRPercentilesAggregator.java | 9 +-- .../AbstractTDigestPercentilesAggregator.java | 9 +-- .../metrics/stats/StatsAggregator.java | 29 ++++---- .../extended/ExtendedStatsAggregator.java | 35 +++++----- .../metrics/sum/SumAggregator.java | 13 ++-- .../valuecount/ValueCountAggregator.java | 5 +- 15 files changed, 173 insertions(+), 156 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridAggregator.java index d20b216cd0748..ebcb7b39dbad4 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridAggregator.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridAggregator.java @@ -69,21 +69,22 @@ public LeafBucketCollector getLeafCollector(LeafReaderContext ctx, @Override public void collect(int doc, long bucket) throws IOException { assert bucket == 0; - values.setDocument(doc); - final int valuesCount = values.count(); - - long previous = Long.MAX_VALUE; - for (int i = 0; i < valuesCount; ++i) { - final long val = values.valueAt(i); - if (previous != val || i == 0) { - long bucketOrdinal = bucketOrds.add(val); - if (bucketOrdinal < 0) { // already seen - bucketOrdinal = - 1 - bucketOrdinal; - collectExistingBucket(sub, doc, bucketOrdinal); - } else { - collectBucket(sub, doc, bucketOrdinal); + if (values.advanceExact(doc)) { + final int valuesCount = values.docValueCount(); + + long previous = Long.MAX_VALUE; + for (int i = 0; i < valuesCount; ++i) { + final long val = values.nextValue(); + if (previous != val || i == 0) { + long bucketOrdinal = bucketOrds.add(val); + if (bucketOrdinal < 0) { // already seen + bucketOrdinal = -1 - bucketOrdinal; + collectExistingBucket(sub, doc, bucketOrdinal); + } else { + collectBucket(sub, doc, bucketOrdinal); + } + previous = val; } - previous = val; } } } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregator.java index a52f2b2cfb5aa..871cade50cd71 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregator.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregator.java @@ -97,25 +97,26 @@ public LeafBucketCollector getLeafCollector(LeafReaderContext ctx, @Override public void collect(int doc, long bucket) throws IOException { assert bucket == 0; - values.setDocument(doc); - final int valuesCount = values.count(); - - long previousRounded = Long.MIN_VALUE; - for (int i = 0; i < valuesCount; ++i) { - long value = values.valueAt(i); - long rounded = rounding.round(value - offset) + offset; - assert rounded >= previousRounded; - if (rounded == previousRounded) { - continue; + if (values.advanceExact(doc)) { + final int valuesCount = values.docValueCount(); + + long previousRounded = Long.MIN_VALUE; + for (int i = 0; i < valuesCount; ++i) { + long value = values.nextValue(); + long rounded = rounding.round(value - offset) + offset; + assert rounded >= previousRounded; + if (rounded == previousRounded) { + continue; + } + long bucketOrd = bucketOrds.add(rounded); + if (bucketOrd < 0) { // already seen + bucketOrd = -1 - bucketOrd; + collectExistingBucket(sub, doc, bucketOrd); + } else { + collectBucket(sub, doc, bucketOrd); + } + previousRounded = rounded; } - long bucketOrd = bucketOrds.add(rounded); - if (bucketOrd < 0) { // already seen - bucketOrd = -1 - bucketOrd; - collectExistingBucket(sub, doc, bucketOrd); - } else { - collectBucket(sub, doc, bucketOrd); - } - previousRounded = rounded; } } }; diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/HistogramAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/HistogramAggregator.java index 4b547989d8bce..a3a038cfa3cae 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/HistogramAggregator.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/HistogramAggregator.java @@ -101,25 +101,26 @@ public LeafBucketCollector getLeafCollector(LeafReaderContext ctx, @Override public void collect(int doc, long bucket) throws IOException { assert bucket == 0; - values.setDocument(doc); - final int valuesCount = values.count(); - - double previousKey = Double.NEGATIVE_INFINITY; - for (int i = 0; i < valuesCount; ++i) { - double value = values.valueAt(i); - double key = Math.floor((value - offset) / interval); - assert key >= previousKey; - if (key == previousKey) { - continue; + if (values.advanceExact(doc)) { + final int valuesCount = values.docValueCount(); + + double previousKey = Double.NEGATIVE_INFINITY; + for (int i = 0; i < valuesCount; ++i) { + double value = values.nextValue(); + double key = Math.floor((value - offset) / interval); + assert key >= previousKey; + if (key == previousKey) { + continue; + } + long bucketOrd = bucketOrds.add(Double.doubleToLongBits(key)); + if (bucketOrd < 0) { // already seen + bucketOrd = -1 - bucketOrd; + collectExistingBucket(sub, doc, bucketOrd); + } else { + collectBucket(sub, doc, bucketOrd); + } + previousKey = key; } - long bucketOrd = bucketOrds.add(Double.doubleToLongBits(key)); - if (bucketOrd < 0) { // already seen - bucketOrd = -1 - bucketOrd; - collectExistingBucket(sub, doc, bucketOrd); - } else { - collectBucket(sub, doc, bucketOrd); - } - previousKey = key; } } }; diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregator.java index a2e55c18fb7d2..45128030759e7 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregator.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregator.java @@ -236,11 +236,12 @@ public LeafBucketCollector getLeafCollector(LeafReaderContext ctx, return new LeafBucketCollectorBase(sub, values) { @Override public void collect(int doc, long bucket) throws IOException { - values.setDocument(doc); - final int valuesCount = values.count(); - for (int i = 0, lo = 0; i < valuesCount; ++i) { - final double value = values.valueAt(i); - lo = collect(doc, value, bucket, lo); + if (values.advanceExact(doc)) { + final int valuesCount = values.docValueCount(); + for (int i = 0, lo = 0; i < valuesCount; ++i) { + final double value = values.nextValue(); + lo = collect(doc, value, bucket, lo); + } } } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/avg/AvgAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/avg/AvgAggregator.java index eb75bae268f19..0decfa05575e4 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/avg/AvgAggregator.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/avg/AvgAggregator.java @@ -77,14 +77,15 @@ public void collect(int doc, long bucket) throws IOException { counts = bigArrays.grow(counts, bucket + 1); sums = bigArrays.grow(sums, bucket + 1); - values.setDocument(doc); - final int valueCount = values.count(); - counts.increment(bucket, valueCount); - double sum = 0; - for (int i = 0; i < valueCount; i++) { - sum += values.valueAt(i); + if (values.advanceExact(doc)) { + final int valueCount = values.docValueCount(); + counts.increment(bucket, valueCount); + double sum = 0; + for (int i = 0; i < valueCount; i++) { + sum += values.nextValue(); + } + sums.increment(bucket, sum); } - sums.increment(bucket, sum); } }; } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geobounds/GeoBoundsAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geobounds/GeoBoundsAggregator.java index 2083ea570d30a..5c0cb4ba60a1f 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geobounds/GeoBoundsAggregator.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geobounds/GeoBoundsAggregator.java @@ -102,41 +102,42 @@ public void collect(int doc, long bucket) throws IOException { negRights.fill(from, negRights.size(), Double.NEGATIVE_INFINITY); } - values.setDocument(doc); - final int valuesCount = values.count(); + if (values.advanceExact(doc)) { + final int valuesCount = values.docValueCount(); - for (int i = 0; i < valuesCount; ++i) { - GeoPoint value = values.valueAt(i); - double top = tops.get(bucket); - if (value.lat() > top) { - top = value.lat(); + for (int i = 0; i < valuesCount; ++i) { + GeoPoint value = values.nextValue(); + double top = tops.get(bucket); + if (value.lat() > top) { + top = value.lat(); + } + double bottom = bottoms.get(bucket); + if (value.lat() < bottom) { + bottom = value.lat(); + } + double posLeft = posLefts.get(bucket); + if (value.lon() >= 0 && value.lon() < posLeft) { + posLeft = value.lon(); + } + double posRight = posRights.get(bucket); + if (value.lon() >= 0 && value.lon() > posRight) { + posRight = value.lon(); + } + double negLeft = negLefts.get(bucket); + if (value.lon() < 0 && value.lon() < negLeft) { + negLeft = value.lon(); + } + double negRight = negRights.get(bucket); + if (value.lon() < 0 && value.lon() > negRight) { + negRight = value.lon(); + } + tops.set(bucket, top); + bottoms.set(bucket, bottom); + posLefts.set(bucket, posLeft); + posRights.set(bucket, posRight); + negLefts.set(bucket, negLeft); + negRights.set(bucket, negRight); } - double bottom = bottoms.get(bucket); - if (value.lat() < bottom) { - bottom = value.lat(); - } - double posLeft = posLefts.get(bucket); - if (value.lon() >= 0 && value.lon() < posLeft) { - posLeft = value.lon(); - } - double posRight = posRights.get(bucket); - if (value.lon() >= 0 && value.lon() > posRight) { - posRight = value.lon(); - } - double negLeft = negLefts.get(bucket); - if (value.lon() < 0 && value.lon() < negLeft) { - negLeft = value.lon(); - } - double negRight = negRights.get(bucket); - if (value.lon() < 0 && value.lon() > negRight) { - negRight = value.lon(); - } - tops.set(bucket, top); - bottoms.set(bucket, bottom); - posLefts.set(bucket, posLeft); - posRights.set(bucket, posRight); - negLefts.set(bucket, negLeft); - negRights.set(bucket, negRight); } } }; diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/GeoCentroidAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/GeoCentroidAggregator.java index 9d537030fa948..c6a5326a773b3 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/GeoCentroidAggregator.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/GeoCentroidAggregator.java @@ -72,15 +72,15 @@ public void collect(int doc, long bucket) throws IOException { centroids = bigArrays.grow(centroids, bucket + 1); counts = bigArrays.grow(counts, bucket + 1); - values.setDocument(doc); - final int valueCount = values.count(); - if (valueCount > 0) { + if (values.advanceExact(doc)) { + final int valueCount = values.docValueCount(); double[] pt = new double[2]; // get the previously accumulated number of counts long prevCounts = counts.get(bucket); // increment by the number of points for this document counts.increment(bucket, valueCount); - // get the previous GeoPoint if a moving avg was computed + // get the previous GeoPoint if a moving avg was + // computed if (prevCounts > 0) { final long mortonCode = centroids.get(bucket); pt[0] = GeoPointField.decodeLongitude(mortonCode); @@ -88,11 +88,12 @@ public void collect(int doc, long bucket) throws IOException { } // update the moving average for (int i = 0; i < valueCount; ++i) { - GeoPoint value = values.valueAt(i); + GeoPoint value = values.nextValue(); pt[0] = pt[0] + (value.getLon() - pt[0]) / ++prevCounts; pt[1] = pt[1] + (value.getLat() - pt[1]) / prevCounts; } - // TODO: we do not need to interleave the lat and lon bits here + // TODO: we do not need to interleave the lat and lon + // bits here // should we just store them contiguously? centroids.set(bucket, GeoPointField.encodeLatLon(pt[1], pt[0])); } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/max/MaxAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/max/MaxAggregator.java index cef7f373280b3..8ef4d0b7e29d1 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/max/MaxAggregator.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/max/MaxAggregator.java @@ -82,10 +82,12 @@ public void collect(int doc, long bucket) throws IOException { maxes = bigArrays.grow(maxes, bucket + 1); maxes.fill(from, maxes.size(), Double.NEGATIVE_INFINITY); } - final double value = values.get(doc); - double max = maxes.get(bucket); - max = Math.max(max, value); - maxes.set(bucket, max); + if (values.advanceExact(doc)) { + final double value = values.doubleValue(); + double max = maxes.get(bucket); + max = Math.max(max, value); + maxes.set(bucket, max); + } } }; diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/min/MinAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/min/MinAggregator.java index 31e87b12a6251..f355f55139c04 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/min/MinAggregator.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/min/MinAggregator.java @@ -81,10 +81,12 @@ public void collect(int doc, long bucket) throws IOException { mins = bigArrays.grow(mins, bucket + 1); mins.fill(from, mins.size(), Double.POSITIVE_INFINITY); } - final double value = values.get(doc); - double min = mins.get(bucket); - min = Math.min(min, value); - mins.set(bucket, min); + if (values.advanceExact(doc)) { + final double value = values.doubleValue(); + double min = mins.get(bucket); + min = Math.min(min, value); + mins.set(bucket, min); + } } }; diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/AbstractHDRPercentilesAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/AbstractHDRPercentilesAggregator.java index bf4443c88739f..47c267aae903e 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/AbstractHDRPercentilesAggregator.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/AbstractHDRPercentilesAggregator.java @@ -95,10 +95,11 @@ public void collect(int doc, long bucket) throws IOException { states.set(bucket, state); } - values.setDocument(doc); - final int valueCount = values.count(); - for (int i = 0; i < valueCount; i++) { - state.recordValue(values.valueAt(i)); + if (values.advanceExact(doc)) { + final int valueCount = values.docValueCount(); + for (int i = 0; i < valueCount; i++) { + state.recordValue(values.nextValue()); + } } } }; diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/AbstractTDigestPercentilesAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/AbstractTDigestPercentilesAggregator.java index 2c68d580e14b5..1b5ed510f8d61 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/AbstractTDigestPercentilesAggregator.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/AbstractTDigestPercentilesAggregator.java @@ -87,10 +87,11 @@ public void collect(int doc, long bucket) throws IOException { states.set(bucket, state); } - values.setDocument(doc); - final int valueCount = values.count(); - for (int i = 0; i < valueCount; i++) { - state.add(values.valueAt(i)); + if (values.advanceExact(doc)) { + final int valueCount = values.docValueCount(); + for (int i = 0; i < valueCount; i++) { + state.add(values.nextValue()); + } } } }; diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/StatsAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/StatsAggregator.java index e3657c3851d4d..cca176bd1ad5f 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/StatsAggregator.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/StatsAggregator.java @@ -94,21 +94,22 @@ public void collect(int doc, long bucket) throws IOException { maxes.fill(from, overSize, Double.NEGATIVE_INFINITY); } - values.setDocument(doc); - final int valuesCount = values.count(); - counts.increment(bucket, valuesCount); - double sum = 0; - double min = mins.get(bucket); - double max = maxes.get(bucket); - for (int i = 0; i < valuesCount; i++) { - double value = values.valueAt(i); - sum += value; - min = Math.min(min, value); - max = Math.max(max, value); + if (values.advanceExact(doc)) { + final int valuesCount = values.docValueCount(); + counts.increment(bucket, valuesCount); + double sum = 0; + double min = mins.get(bucket); + double max = maxes.get(bucket); + for (int i = 0; i < valuesCount; i++) { + double value = values.nextValue(); + sum += value; + min = Math.min(min, value); + max = Math.max(max, value); + } + sums.increment(bucket, sum); + mins.set(bucket, min); + maxes.set(bucket, max); } - sums.increment(bucket, sum); - mins.set(bucket, min); - maxes.set(bucket, max); } }; } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/extended/ExtendedStatsAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/extended/ExtendedStatsAggregator.java index d6faf5cbb78a0..8dd78bf13730b 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/extended/ExtendedStatsAggregator.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/extended/ExtendedStatsAggregator.java @@ -102,24 +102,25 @@ public void collect(int doc, long bucket) throws IOException { maxes.fill(from, overSize, Double.NEGATIVE_INFINITY); } - values.setDocument(doc); - final int valuesCount = values.count(); - counts.increment(bucket, valuesCount); - double sum = 0; - double sumOfSqr = 0; - double min = mins.get(bucket); - double max = maxes.get(bucket); - for (int i = 0; i < valuesCount; i++) { - double value = values.valueAt(i); - sum += value; - sumOfSqr += value * value; - min = Math.min(min, value); - max = Math.max(max, value); + if (values.advanceExact(doc)) { + final int valuesCount = values.docValueCount(); + counts.increment(bucket, valuesCount); + double sum = 0; + double sumOfSqr = 0; + double min = mins.get(bucket); + double max = maxes.get(bucket); + for (int i = 0; i < valuesCount; i++) { + double value = values.nextValue(); + sum += value; + sumOfSqr += value * value; + min = Math.min(min, value); + max = Math.max(max, value); + } + sums.increment(bucket, sum); + sumOfSqrs.increment(bucket, sumOfSqr); + mins.set(bucket, min); + maxes.set(bucket, max); } - sums.increment(bucket, sum); - sumOfSqrs.increment(bucket, sumOfSqr); - mins.set(bucket, min); - maxes.set(bucket, max); } }; diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/sum/SumAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/sum/SumAggregator.java index f7f2ee5501bce..bd325b39373e5 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/sum/SumAggregator.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/sum/SumAggregator.java @@ -71,13 +71,14 @@ public LeafBucketCollector getLeafCollector(LeafReaderContext ctx, @Override public void collect(int doc, long bucket) throws IOException { sums = bigArrays.grow(sums, bucket + 1); - values.setDocument(doc); - final int valuesCount = values.count(); - double sum = 0; - for (int i = 0; i < valuesCount; i++) { - sum += values.valueAt(i); + if (values.advanceExact(doc)) { + final int valuesCount = values.docValueCount(); + double sum = 0; + for (int i = 0; i < valuesCount; i++) { + sum += values.nextValue(); + } + sums.increment(bucket, sum); } - sums.increment(bucket, sum); } }; } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/valuecount/ValueCountAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/valuecount/ValueCountAggregator.java index e0ed6d1ade067..929e26d04ba77 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/valuecount/ValueCountAggregator.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/valuecount/ValueCountAggregator.java @@ -73,8 +73,9 @@ public LeafBucketCollector getLeafCollector(LeafReaderContext ctx, @Override public void collect(int doc, long bucket) throws IOException { counts = bigArrays.grow(counts, bucket + 1); - values.setDocument(doc); - counts.increment(bucket, values.count()); + if (values.advanceExact(doc)) { + counts.increment(bucket, values.docValueCount()); + } } }; From b7279c07efc4ca9037b444cc87d3fbc9331dfca7 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Wed, 5 Apr 2017 14:26:36 +0200 Subject: [PATCH 07/87] Fix compile errors in MultiValueMode. --- .../fielddata/AbstractBinaryDocValues.java | 55 ++ .../fielddata/AbstractNumericDocValues.java | 55 ++ ...Ords.java => AbstractSortedDocValues.java} | 35 +- .../AbstractSortedNumericDocValues.java | 55 ++ .../elasticsearch/search/MultiValueMode.java | 852 +++++++++--------- 5 files changed, 598 insertions(+), 454 deletions(-) create mode 100644 core/src/main/java/org/elasticsearch/index/fielddata/AbstractBinaryDocValues.java create mode 100644 core/src/main/java/org/elasticsearch/index/fielddata/AbstractNumericDocValues.java rename core/src/main/java/org/elasticsearch/index/fielddata/{AbstractRandomAccessOrds.java => AbstractSortedDocValues.java} (51%) create mode 100644 core/src/main/java/org/elasticsearch/index/fielddata/AbstractSortedNumericDocValues.java diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/AbstractBinaryDocValues.java b/core/src/main/java/org/elasticsearch/index/fielddata/AbstractBinaryDocValues.java new file mode 100644 index 0000000000000..a3ce33a4e6d21 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/index/fielddata/AbstractBinaryDocValues.java @@ -0,0 +1,55 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.fielddata; + +import org.apache.lucene.index.BinaryDocValues; +import org.apache.lucene.search.DocIdSetIterator; + +import java.io.IOException; + +/** + * Base implementation that throws an {@link IOException} for the + * {@link DocIdSetIterator} APIs. This impl is safe to use for sorting and + * aggregations, which only use {@link #advanceExact(int)} and + * {@link #binaryValue()}. + */ +public abstract class AbstractBinaryDocValues extends BinaryDocValues { + + @Override + public int docID() { + throw new UnsupportedOperationException(); + } + + @Override + public int nextDoc() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public int advance(int target) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public long cost() { + throw new UnsupportedOperationException(); + } + +} diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/AbstractNumericDocValues.java b/core/src/main/java/org/elasticsearch/index/fielddata/AbstractNumericDocValues.java new file mode 100644 index 0000000000000..effefa3bdb3e4 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/index/fielddata/AbstractNumericDocValues.java @@ -0,0 +1,55 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.fielddata; + +import org.apache.lucene.index.NumericDocValues; +import org.apache.lucene.search.DocIdSetIterator; + +import java.io.IOException; + +/** + * Base implementation that throws an {@link IOException} for the + * {@link DocIdSetIterator} APIs. This impl is safe to use for sorting and + * aggregations, which only use {@link #advanceExact(int)} and + * {@link #longValue()}. + */ +public abstract class AbstractNumericDocValues extends NumericDocValues { + + @Override + public int docID() { + throw new UnsupportedOperationException(); + } + + @Override + public int nextDoc() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public int advance(int target) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public long cost() { + throw new UnsupportedOperationException(); + } + +} diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/AbstractRandomAccessOrds.java b/core/src/main/java/org/elasticsearch/index/fielddata/AbstractSortedDocValues.java similarity index 51% rename from core/src/main/java/org/elasticsearch/index/fielddata/AbstractRandomAccessOrds.java rename to core/src/main/java/org/elasticsearch/index/fielddata/AbstractSortedDocValues.java index ef8954b97e14b..1cf2bcf999161 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/AbstractRandomAccessOrds.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/AbstractSortedDocValues.java @@ -19,30 +19,37 @@ package org.elasticsearch.index.fielddata; -import org.apache.lucene.index.RandomAccessOrds; +import org.apache.lucene.index.SortedDocValues; +import org.apache.lucene.search.DocIdSetIterator; + +import java.io.IOException; /** - * Base implementation of a {@link RandomAccessOrds} instance. + * Base implementation that throws an {@link IOException} for the + * {@link DocIdSetIterator} APIs. This impl is safe to use for sorting and + * aggregations, which only use {@link #advanceExact(int)} and + * {@link #ordValue()}. */ -public abstract class AbstractRandomAccessOrds extends RandomAccessOrds { +public abstract class AbstractSortedDocValues extends SortedDocValues { - int i = 0; + @Override + public int docID() { + throw new UnsupportedOperationException(); + } - protected abstract void doSetDocument(int docID); + @Override + public int nextDoc() throws IOException { + throw new UnsupportedOperationException(); + } @Override - public final void setDocument(int docID) { - doSetDocument(docID); - i = 0; + public int advance(int target) throws IOException { + throw new UnsupportedOperationException(); } @Override - public long nextOrd() { - if (i < cardinality()) { - return ordAt(i++); - } else { - return NO_MORE_ORDS; - } + public long cost() { + throw new UnsupportedOperationException(); } } diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/AbstractSortedNumericDocValues.java b/core/src/main/java/org/elasticsearch/index/fielddata/AbstractSortedNumericDocValues.java new file mode 100644 index 0000000000000..c7f78147797e3 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/index/fielddata/AbstractSortedNumericDocValues.java @@ -0,0 +1,55 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.fielddata; + +import org.apache.lucene.index.SortedNumericDocValues; +import org.apache.lucene.search.DocIdSetIterator; + +import java.io.IOException; + +/** + * Base implementation that throws an {@link IOException} for the + * {@link DocIdSetIterator} APIs. This impl is safe to use for sorting and + * aggregations, which only use {@link #advanceExact(int)} and + * {@link #docValueCount()} and {@link #nextValue()}. + */ +public abstract class AbstractSortedNumericDocValues extends SortedNumericDocValues { + + @Override + public int docID() { + throw new UnsupportedOperationException(); + } + + @Override + public int nextDoc() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public int advance(int target) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public long cost() { + throw new UnsupportedOperationException(); + } + +} diff --git a/core/src/main/java/org/elasticsearch/search/MultiValueMode.java b/core/src/main/java/org/elasticsearch/search/MultiValueMode.java index 90e3417ad1f11..4fab51dcb81ad 100644 --- a/core/src/main/java/org/elasticsearch/search/MultiValueMode.java +++ b/core/src/main/java/org/elasticsearch/search/MultiValueMode.java @@ -23,17 +23,19 @@ import org.apache.lucene.index.BinaryDocValues; import org.apache.lucene.index.DocValues; import org.apache.lucene.index.NumericDocValues; -import org.apache.lucene.index.RandomAccessOrds; import org.apache.lucene.index.SortedDocValues; import org.apache.lucene.index.SortedNumericDocValues; +import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.util.BitSet; -import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRefBuilder; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.index.fielddata.AbstractBinaryDocValues; +import org.elasticsearch.index.fielddata.AbstractNumericDocValues; +import org.elasticsearch.index.fielddata.AbstractSortedDocValues; import org.elasticsearch.index.fielddata.FieldData; import org.elasticsearch.index.fielddata.NumericDoubleValues; import org.elasticsearch.index.fielddata.SortedBinaryDocValues; @@ -52,86 +54,65 @@ public enum MultiValueMode implements Writeable { */ SUM { @Override - protected long pick(SortedNumericDocValues values, long missingValue, int doc) { - values.setDocument(doc); - final int count = values.count(); - if (count > 0) { - long total = 0; - for (int index = 0; index < count; ++index) { - total += values.valueAt(index); - } - return total; - } else { - return missingValue; + protected long pick(SortedNumericDocValues values) throws IOException { + final int count = values.docValueCount(); + long total = 0; + for (int index = 0; index < count; ++index) { + total += values.nextValue(); } + return total; } @Override - protected long pick(SortedNumericDocValues values, long missingValue, DocIdSetIterator docItr, int startDoc, int endDoc) { - try { - int totalCount = 0; - long totalValue = 0; - for (int doc = startDoc; doc < endDoc; doc = docItr.nextDoc()) { - values.setDocument(doc); - final int count = values.count(); + protected long pick(SortedNumericDocValues values, long missingValue, DocIdSetIterator docItr, int startDoc, int endDoc) throws IOException { + int totalCount = 0; + long totalValue = 0; + for (int doc = startDoc; doc < endDoc; doc = docItr.nextDoc()) { + if (values.advanceExact(doc)) { + final int count = values.docValueCount(); for (int index = 0; index < count; ++index) { - totalValue += values.valueAt(index); + totalValue += values.nextValue(); } totalCount += count; } - return totalCount > 0 ? totalValue : missingValue; - } catch (IOException ioException) { - throw new RuntimeException(ioException); } + return totalCount > 0 ? totalValue : missingValue; } @Override - protected double pick(SortedNumericDoubleValues values, double missingValue, int doc) { - values.setDocument(doc); - final int count = values.count(); - if (count > 0) { - double total = 0; - for (int index = 0; index < count; ++index) { - total += values.valueAt(index); - } - return total; - } else { - return missingValue; + protected double pick(SortedNumericDoubleValues values) throws IOException { + final int count = values.docValueCount(); + double total = 0; + for (int index = 0; index < count; ++index) { + total += values.nextValue(); } + return total; } @Override - protected double pick(SortedNumericDoubleValues values, double missingValue, DocIdSetIterator docItr, int startDoc, int endDoc) { - try { - int totalCount = 0; - double totalValue = 0; - for (int doc = startDoc; doc < endDoc; doc = docItr.nextDoc()) { - values.setDocument(doc); - final int count = values.count(); + protected double pick(SortedNumericDoubleValues values, double missingValue, DocIdSetIterator docItr, int startDoc, int endDoc) throws IOException { + int totalCount = 0; + double totalValue = 0; + for (int doc = startDoc; doc < endDoc; doc = docItr.nextDoc()) { + if (values.advanceExact(doc)) { + final int count = values.docValueCount(); for (int index = 0; index < count; ++index) { - totalValue += values.valueAt(index); + totalValue += values.nextValue(); } totalCount += count; } - return totalCount > 0 ? totalValue : missingValue; - } catch (IOException ioException) { - throw new RuntimeException(ioException); } + return totalCount > 0 ? totalValue : missingValue; } @Override - protected double pick(UnsortedNumericDoubleValues values, double missingValue, int doc) { - values.setDocument(doc); - final int count = values.count(); - if (count > 0) { - double total = 0; - for (int index = 0; index < count; ++index) { - total += values.valueAt(index); - } - return total; - } else { - return missingValue; + protected double pick(UnsortedNumericDoubleValues values) throws IOException { + final int count = values.docValueCount(); + double total = 0; + for (int index = 0; index < count; ++index) { + total += values.nextValue(); } + return total; } }, @@ -140,92 +121,71 @@ protected double pick(UnsortedNumericDoubleValues values, double missingValue, i */ AVG { @Override - protected long pick(SortedNumericDocValues values, long missingValue, int doc) { - values.setDocument(doc); - final int count = values.count(); - if (count > 0) { - long total = 0; - for (int index = 0; index < count; ++index) { - total += values.valueAt(index); - } - return count > 1 ? Math.round((double)total/(double)count) : total; - } else { - return missingValue; + protected long pick(SortedNumericDocValues values) throws IOException { + final int count = values.docValueCount(); + long total = 0; + for (int index = 0; index < count; ++index) { + total += values.nextValue(); } + return count > 1 ? Math.round((double)total/(double)count) : total; } @Override - protected long pick(SortedNumericDocValues values, long missingValue, DocIdSetIterator docItr, int startDoc, int endDoc) { - try { - int totalCount = 0; - long totalValue = 0; - for (int doc = startDoc; doc < endDoc; doc = docItr.nextDoc()) { - values.setDocument(doc); - final int count = values.count(); + protected long pick(SortedNumericDocValues values, long missingValue, DocIdSetIterator docItr, int startDoc, int endDoc) throws IOException { + int totalCount = 0; + long totalValue = 0; + for (int doc = startDoc; doc < endDoc; doc = docItr.nextDoc()) { + if (values.advanceExact(doc)) { + final int count = values.docValueCount(); for (int index = 0; index < count; ++index) { - totalValue += values.valueAt(index); + totalValue += values.nextValue(); } totalCount += count; } - if (totalCount < 1) { - return missingValue; - } - return totalCount > 1 ? Math.round((double)totalValue/(double)totalCount) : totalValue; - } catch (IOException ioException) { - throw new RuntimeException(ioException); } + if (totalCount < 1) { + return missingValue; + } + return totalCount > 1 ? Math.round((double)totalValue/(double)totalCount) : totalValue; } @Override - protected double pick(SortedNumericDoubleValues values, double missingValue, int doc) { - values.setDocument(doc); - final int count = values.count(); - if (count > 0) { - double total = 0; - for (int index = 0; index < count; ++index) { - total += values.valueAt(index); - } - return total/count; - } else { - return missingValue; + protected double pick(SortedNumericDoubleValues values) throws IOException { + final int count = values.docValueCount(); + double total = 0; + for (int index = 0; index < count; ++index) { + total += values.nextValue(); } + return total/count; } @Override - protected double pick(SortedNumericDoubleValues values, double missingValue, DocIdSetIterator docItr, int startDoc, int endDoc) { - try { - int totalCount = 0; - double totalValue = 0; - for (int doc = startDoc; doc < endDoc; doc = docItr.nextDoc()) { - values.setDocument(doc); - final int count = values.count(); + protected double pick(SortedNumericDoubleValues values, double missingValue, DocIdSetIterator docItr, int startDoc, int endDoc) throws IOException { + int totalCount = 0; + double totalValue = 0; + for (int doc = startDoc; doc < endDoc; doc = docItr.nextDoc()) { + if (values.advanceExact(doc)) { + final int count = values.docValueCount(); for (int index = 0; index < count; ++index) { - totalValue += values.valueAt(index); + totalValue += values.nextValue(); } totalCount += count; } - if (totalCount < 1) { - return missingValue; - } - return totalValue/totalCount; - } catch (IOException ioException) { - throw new RuntimeException(ioException); } + if (totalCount < 1) { + return missingValue; + } + return totalValue/totalCount; } @Override - protected double pick(UnsortedNumericDoubleValues values, double missingValue, int doc) { - values.setDocument(doc); - final int count = values.count(); - if (count > 0) { - double total = 0; - for (int index = 0; index < count; ++index) { - total += values.valueAt(index); - } - return total/count; - } else { - return missingValue; + protected double pick(UnsortedNumericDoubleValues values) throws IOException { + final int count = values.docValueCount(); + double total = 0; + for (int index = 0; index < count; ++index) { + total += values.nextValue(); } + return total/count; } }, @@ -234,36 +194,28 @@ protected double pick(UnsortedNumericDoubleValues values, double missingValue, i */ MEDIAN { @Override - protected long pick(SortedNumericDocValues values, long missingValue, int doc) { - values.setDocument(doc); - int count = values.count(); - if (count > 0) { - if (count % 2 == 0) { - count /= 2; - return Math.round((values.valueAt(count - 1) + values.valueAt(count))/2.0); - } else { - count /= 2; - return values.valueAt(count); - } + protected long pick(SortedNumericDocValues values) throws IOException { + int count = values.docValueCount(); + for (int i = 0; i < count / 2; ++i) { + values.nextValue(); + } + if (count % 2 == 0) { + return Math.round(((double) values.nextValue() + values.nextValue()) / 2); } else { - return missingValue; + return values.nextValue(); } } @Override - protected double pick(SortedNumericDoubleValues values, double missingValue, int doc) { - values.setDocument(doc); - int count = values.count(); - if (count > 0) { - if (count % 2 == 0) { - count /= 2; - return (values.valueAt(count - 1) + values.valueAt(count))/2.0; - } else { - count /= 2; - return values.valueAt(count); - } + protected double pick(SortedNumericDoubleValues values) throws IOException { + int count = values.docValueCount(); + for (int i = 0; i < count / 2; ++i) { + values.nextValue(); + } + if (count % 2 == 0) { + return Math.round((values.nextValue() + values.nextValue()) / 2); } else { - return missingValue; + return values.nextValue(); } } }, @@ -273,119 +225,93 @@ protected double pick(SortedNumericDoubleValues values, double missingValue, int */ MIN { @Override - protected long pick(SortedNumericDocValues values, long missingValue, int doc) { - values.setDocument(doc); - final int count = values.count(); - return count > 0 ? values.valueAt(0) : missingValue; + protected long pick(SortedNumericDocValues values) throws IOException { + return values.nextValue(); } @Override - protected long pick(SortedNumericDocValues values, long missingValue, DocIdSetIterator docItr, int startDoc, int endDoc) { - try { - int totalCount = 0; - long minValue = Long.MAX_VALUE; - for (int doc = startDoc; doc < endDoc; doc = docItr.nextDoc()) { - values.setDocument(doc); - final int count = values.count(); - if (count > 0) { - minValue = Math.min(minValue, values.valueAt(0)); - } - totalCount += count; + protected long pick(SortedNumericDocValues values, long missingValue, DocIdSetIterator docItr, int startDoc, int endDoc) throws IOException { + boolean hasValue = false; + long minValue = Long.MAX_VALUE; + for (int doc = startDoc; doc < endDoc; doc = docItr.nextDoc()) { + if (values.advanceExact(doc)) { + minValue = Math.min(minValue, values.nextValue()); + hasValue = true; } - return totalCount > 0 ? minValue : missingValue; - } catch (IOException ioException) { - throw new RuntimeException(ioException); } + return hasValue ? minValue : missingValue; } @Override - protected double pick(SortedNumericDoubleValues values, double missingValue, int doc) { - values.setDocument(doc); - int count = values.count(); - return count > 0 ? values.valueAt(0) : missingValue; + protected double pick(SortedNumericDoubleValues values) throws IOException { + return values.nextValue(); } @Override - protected double pick(SortedNumericDoubleValues values, double missingValue, DocIdSetIterator docItr, int startDoc, int endDoc) { - try { - int totalCount = 0; - double minValue = Double.MAX_VALUE; - for (int doc = startDoc; doc < endDoc; doc = docItr.nextDoc()) { - values.setDocument(doc); - final int count = values.count(); - if (count > 0) { - minValue = Math.min(minValue, values.valueAt(0)); - } - totalCount += count; + protected double pick(SortedNumericDoubleValues values, double missingValue, DocIdSetIterator docItr, int startDoc, int endDoc) throws IOException { + boolean hasValue = false; + double minValue = Double.POSITIVE_INFINITY; + for (int doc = startDoc; doc < endDoc; doc = docItr.nextDoc()) { + if (values.advanceExact(doc)) { + minValue = Math.min(minValue, values.nextValue()); + hasValue = true; } - return totalCount > 0 ? minValue : missingValue; - } catch (IOException ioException) { - throw new RuntimeException(ioException); } + return hasValue ? minValue : missingValue; } @Override - protected BytesRef pick(SortedBinaryDocValues values, BytesRef missingValue, int doc) { - values.setDocument(doc); - final int count = values.count(); - return count > 0 ? values.valueAt(0) : missingValue; + protected BytesRef pick(SortedBinaryDocValues values) throws IOException { + return values.nextValue(); } @Override - protected BytesRef pick(BinaryDocValues values, BytesRefBuilder builder, DocIdSetIterator docItr, int startDoc, int endDoc) { - try { - BytesRefBuilder value = null; - for (int doc = startDoc; doc < endDoc; doc = docItr.nextDoc()) { - final BytesRef innerValue = values.get(doc); - if (innerValue != null) { - if (value == null) { - builder.copyBytes(innerValue); - value = builder; - } else { - final BytesRef min = value.get().compareTo(innerValue) <= 0 ? value.get() : innerValue; - if (min == innerValue) { - value.copyBytes(min); - } + protected BytesRef pick(BinaryDocValues values, BytesRefBuilder builder, DocIdSetIterator docItr, int startDoc, int endDoc) throws IOException { + BytesRefBuilder value = null; + for (int doc = startDoc; doc < endDoc; doc = docItr.nextDoc()) { + if (values.advanceExact(doc)) { + final BytesRef innerValue = values.binaryValue(); + if (value == null) { + builder.copyBytes(innerValue); + value = builder; + } else { + final BytesRef min = value.get().compareTo(innerValue) <= 0 ? value.get() : innerValue; + if (min == innerValue) { + value.copyBytes(min); } } } - return value == null ? null : value.get(); - } catch (IOException ioException) { - throw new RuntimeException(ioException); } + return value == null ? null : value.get(); } @Override - protected int pick(RandomAccessOrds values, int doc) { - values.setDocument(doc); - return values.cardinality() > 0 ? (int)values.ordAt(0) : -1; + protected int pick(SortedSetDocValues values) throws IOException { + return Math.toIntExact(values.nextOrd()); } @Override - protected int pick(SortedDocValues values, DocIdSetIterator docItr, int startDoc, int endDoc) { - try { - int ord = -1; - for (int doc = startDoc; doc < endDoc; doc = docItr.nextDoc()) { - final int innerOrd = values.getOrd(doc); - if (innerOrd != -1) { - ord = ord == -1 ? innerOrd : Math.min(ord, innerOrd); - } + protected int pick(SortedDocValues values, DocIdSetIterator docItr, int startDoc, int endDoc) throws IOException { + int ord = Integer.MAX_VALUE; + boolean hasValue = false; + for (int doc = startDoc; doc < endDoc; doc = docItr.nextDoc()) { + if (values.advanceExact(doc)) { + final int innerOrd = values.ordValue(); + ord = Math.min(ord, innerOrd); + hasValue = true; } - return ord; - } catch (IOException ioException) { - throw new RuntimeException(ioException); } + return hasValue ? ord : -1; } @Override - protected double pick(UnsortedNumericDoubleValues values, double missingValue, int doc) { - values.setDocument(doc); - int count = values.count(); - double min = Double.MAX_VALUE; + protected double pick(UnsortedNumericDoubleValues values) throws IOException { + int count = values.docValueCount(); + double min = Double.POSITIVE_INFINITY; for (int index = 0; index < count; ++index) { - min = Math.min(values.valueAt(index), min); + min = Math.min(values.nextValue(), min); } - return count > 0 ? min : missingValue; + return min; } }, @@ -394,120 +320,114 @@ protected double pick(UnsortedNumericDoubleValues values, double missingValue, i */ MAX { @Override - protected long pick(SortedNumericDocValues values, long missingValue, int doc) { - values.setDocument(doc); - final int count = values.count(); - return count > 0 ? values.valueAt(count - 1) : missingValue; + protected long pick(SortedNumericDocValues values) throws IOException { + final int count = values.docValueCount(); + for (int i = 0; i < count - 1; ++i) { + values.nextValue(); + } + return values.nextValue(); } @Override - protected long pick(SortedNumericDocValues values, long missingValue, DocIdSetIterator docItr, int startDoc, int endDoc) { - try { - int totalCount = 0; - long maxValue = Long.MIN_VALUE; - for (int doc = startDoc; doc < endDoc; doc = docItr.nextDoc()) { - values.setDocument(doc); - final int count = values.count(); - if (count > 0) { - maxValue = Math.max(maxValue, values.valueAt(count - 1)); + protected long pick(SortedNumericDocValues values, long missingValue, DocIdSetIterator docItr, int startDoc, int endDoc) throws IOException { + boolean hasValue = false; + long maxValue = Long.MIN_VALUE; + for (int doc = startDoc; doc < endDoc; doc = docItr.nextDoc()) { + if (values.advanceExact(doc)) { + final int count = values.docValueCount(); + for (int i = 0; i < count - 1; ++i) { + values.nextValue(); } - totalCount += count; + maxValue = Math.max(maxValue, values.nextValue()); + hasValue = true; } - return totalCount > 0 ? maxValue : missingValue; - } catch (IOException ioException) { - throw new RuntimeException(ioException); } + return hasValue ? maxValue : missingValue; } @Override - protected double pick(SortedNumericDoubleValues values, double missingValue, int doc) { - values.setDocument(doc); - final int count = values.count(); - return count > 0 ? values.valueAt(count - 1) : missingValue; + protected double pick(SortedNumericDoubleValues values) throws IOException { + final int count = values.docValueCount(); + for (int i = 0; i < count - 1; ++i) { + values.nextValue(); + } + return values.nextValue(); } @Override - protected double pick(SortedNumericDoubleValues values, double missingValue, DocIdSetIterator docItr, int startDoc, int endDoc) { - try { - int totalCount = 0; - double maxValue = Double.MIN_VALUE; - for (int doc = startDoc; doc < endDoc; doc = docItr.nextDoc()) { - values.setDocument(doc); - final int count = values.count(); - if (count > 0) { - maxValue = Math.max(maxValue, values.valueAt(count - 1)); + protected double pick(SortedNumericDoubleValues values, double missingValue, DocIdSetIterator docItr, int startDoc, int endDoc) throws IOException { + boolean hasValue = false; + double maxValue = Double.NEGATIVE_INFINITY; + for (int doc = startDoc; doc < endDoc; doc = docItr.nextDoc()) { + if (values.advanceExact(doc)) { + final int count = values.docValueCount(); + for (int i = 0; i < count - 1; ++i) { + values.nextValue(); } - totalCount += count; + maxValue = Math.max(maxValue, values.nextValue()); + hasValue = true; } - return totalCount > 0 ? maxValue : missingValue; - } catch (IOException ioException) { - throw new RuntimeException(ioException); } + return hasValue ? maxValue : missingValue; } @Override - protected BytesRef pick(SortedBinaryDocValues values, BytesRef missingValue, int doc) { - values.setDocument(doc); - final int count = values.count(); - return count > 0 ? values.valueAt(count - 1) : missingValue; + protected BytesRef pick(SortedBinaryDocValues values) throws IOException { + int count = values.docValueCount(); + for (int i = 0; i < count - 1; ++i) { + values.nextValue(); + } + return values.nextValue(); } @Override - protected BytesRef pick(BinaryDocValues values, BytesRefBuilder builder, DocIdSetIterator docItr, int startDoc, int endDoc) { - try { - BytesRefBuilder value = null; - for (int doc = startDoc; doc < endDoc; doc = docItr.nextDoc()) { - final BytesRef innerValue = values.get(doc); - if (innerValue != null) { - if (value == null) { - builder.copyBytes(innerValue); - value = builder; - } else { - final BytesRef max = value.get().compareTo(innerValue) > 0 ? value.get() : innerValue; - if (max == innerValue) { - value.copyBytes(max); - } + protected BytesRef pick(BinaryDocValues values, BytesRefBuilder builder, DocIdSetIterator docItr, int startDoc, int endDoc) throws IOException { + BytesRefBuilder value = null; + for (int doc = startDoc; doc < endDoc; doc = docItr.nextDoc()) { + if (values.advanceExact(doc)) { + final BytesRef innerValue = values.binaryValue(); + if (value == null) { + builder.copyBytes(innerValue); + value = builder; + } else { + final BytesRef max = value.get().compareTo(innerValue) > 0 ? value.get() : innerValue; + if (max == innerValue) { + value.copyBytes(max); } } } - return value == null ? null : value.get(); - } catch (IOException ioException) { - throw new RuntimeException(ioException); } + return value == null ? null : value.get(); } @Override - protected int pick(RandomAccessOrds values, int doc) { - values.setDocument(doc); - final int count = values.cardinality(); - return count > 0 ? (int)values.ordAt(count - 1) : -1; + protected int pick(SortedSetDocValues values) throws IOException { + long maxOrd = -1; + for (long ord = values.nextOrd(); ord != SortedSetDocValues.NO_MORE_ORDS; ord = values.nextOrd()) { + maxOrd = ord; + } + return Math.toIntExact(maxOrd); } @Override - protected int pick(SortedDocValues values, DocIdSetIterator docItr, int startDoc, int endDoc) { - try { - int ord = -1; - for (int doc = startDoc; doc < endDoc; doc = docItr.nextDoc()) { - final int innerOrd = values.getOrd(doc); - if (innerOrd != -1) { - ord = Math.max(ord, innerOrd); - } + protected int pick(SortedDocValues values, DocIdSetIterator docItr, int startDoc, int endDoc) throws IOException { + int ord = -1; + for (int doc = startDoc; doc < endDoc; doc = docItr.nextDoc()) { + if (values.advanceExact(doc)) { + ord = Math.max(ord, values.ordValue()); } - return ord; - } catch (IOException ioException) { - throw new RuntimeException(ioException); } + return ord; } @Override - protected double pick(UnsortedNumericDoubleValues values, double missingValue, int doc) { - values.setDocument(doc); - int count = values.count(); - double max = Double.MIN_VALUE; + protected double pick(UnsortedNumericDoubleValues values) throws IOException { + int count = values.docValueCount(); + double max = Double.NEGATIVE_INFINITY; for (int index = 0; index < count; ++index) { - max = Math.max(values.valueAt(index), max); + max = Math.max(values.nextValue(), max); } - return count > 0 ? max : missingValue; + return max; } }; @@ -534,32 +454,41 @@ public static MultiValueMode fromString(String sortMode) { public NumericDocValues select(final SortedNumericDocValues values, final long missingValue) { final NumericDocValues singleton = DocValues.unwrapSingleton(values); if (singleton != null) { - final Bits docsWithField = DocValues.unwrapSingletonBits(values); - if (docsWithField == null || missingValue == 0) { - return singleton; - } else { - return new NumericDocValues() { - @Override - public long get(int docID) { - final long value = singleton.get(docID); - if (value == 0 && docsWithField.get(docID) == false) { - return missingValue; - } - return value; - } - }; - } + return new AbstractNumericDocValues() { + + private boolean hasValue; + + @Override + public boolean advanceExact(int target) throws IOException { + hasValue = singleton.advanceExact(target); + return true; + } + + @Override + public long longValue() throws IOException { + return hasValue ? singleton.longValue() : missingValue; + } + }; } else { - return new NumericDocValues() { + return new AbstractNumericDocValues() { + + private boolean hasValue; + @Override - public long get(int docID) { - return pick(values, missingValue, docID); + public boolean advanceExact(int target) throws IOException { + hasValue = values.advanceExact(target); + return true; + } + + @Override + public long longValue() throws IOException { + return hasValue ? pick(values) : missingValue; } }; } } - protected long pick(SortedNumericDocValues values, long missingValue, int doc) { + protected long pick(SortedNumericDocValues values) throws IOException { throw new IllegalArgumentException("Unsupported sort mode: " + this); } @@ -580,38 +509,39 @@ public NumericDocValues select(final SortedNumericDocValues values, final long m return select(DocValues.emptySortedNumeric(maxDoc), missingValue); } - return new NumericDocValues() { + return new AbstractNumericDocValues() { int lastSeenRootDoc = 0; long lastEmittedValue = missingValue; @Override - public long get(int rootDoc) { + public boolean advanceExact(int rootDoc) throws IOException { assert rootDocs.get(rootDoc) : "can only sort root documents"; assert rootDoc >= lastSeenRootDoc : "can only evaluate current and upcoming root docs"; if (rootDoc == lastSeenRootDoc) { - return lastEmittedValue; + return true; } - try { - final int prevRootDoc = rootDocs.prevSetBit(rootDoc - 1); - final int firstNestedDoc; - if (innerDocs.docID() > prevRootDoc) { - firstNestedDoc = innerDocs.docID(); - } else { - firstNestedDoc = innerDocs.advance(prevRootDoc + 1); - } - - lastSeenRootDoc = rootDoc; - lastEmittedValue = pick(values, missingValue, innerDocs, firstNestedDoc, rootDoc); - return lastEmittedValue; - } catch (IOException e) { - throw new RuntimeException(e); + final int prevRootDoc = rootDocs.prevSetBit(rootDoc - 1); + final int firstNestedDoc; + if (innerDocs.docID() > prevRootDoc) { + firstNestedDoc = innerDocs.docID(); + } else { + firstNestedDoc = innerDocs.advance(prevRootDoc + 1); } + + lastSeenRootDoc = rootDoc; + lastEmittedValue = pick(values, missingValue, innerDocs, firstNestedDoc, rootDoc); + return true; + } + + @Override + public long longValue() { + return lastEmittedValue; } }; } - protected long pick(SortedNumericDocValues values, long missingValue, DocIdSetIterator docItr, int startDoc, int endDoc) { + protected long pick(SortedNumericDocValues values, long missingValue, DocIdSetIterator docItr, int startDoc, int endDoc) throws IOException { throw new IllegalArgumentException("Unsupported sort mode: " + this); } @@ -625,32 +555,42 @@ protected long pick(SortedNumericDocValues values, long missingValue, DocIdSetIt public NumericDoubleValues select(final SortedNumericDoubleValues values, final double missingValue) { final NumericDoubleValues singleton = FieldData.unwrapSingleton(values); if (singleton != null) { - final Bits docsWithField = FieldData.unwrapSingletonBits(values); - if (docsWithField == null || missingValue == 0) { - return singleton; - } else { - return new NumericDoubleValues() { - @Override - public double get(int docID) { - final double value = singleton.get(docID); - if (value == 0 && docsWithField.get(docID) == false) { - return missingValue; - } - return value; - } - }; - } + return new NumericDoubleValues() { + + private boolean hasValue; + + @Override + public boolean advanceExact(int doc) throws IOException { + hasValue = singleton.advanceExact(doc); + return true; + } + + @Override + public double doubleValue() throws IOException { + return hasValue ? singleton.doubleValue() : missingValue; + } + + }; } else { return new NumericDoubleValues() { + + private boolean hasValue; + @Override - public double get(int docID) { - return pick(values, missingValue, docID); + public boolean advanceExact(int target) throws IOException { + hasValue = values.advanceExact(target); + return true; + } + + @Override + public double doubleValue() throws IOException { + return hasValue ? pick(values) : missingValue; } }; } } - protected double pick(SortedNumericDoubleValues values, double missingValue, int doc) { + protected double pick(SortedNumericDoubleValues values) throws IOException { throw new IllegalArgumentException("Unsupported sort mode: " + this); } @@ -668,7 +608,7 @@ protected double pick(SortedNumericDoubleValues values, double missingValue, int */ public NumericDoubleValues select(final SortedNumericDoubleValues values, final double missingValue, final BitSet rootDocs, final DocIdSetIterator innerDocs, int maxDoc) throws IOException { if (rootDocs == null || innerDocs == null) { - return select(FieldData.emptySortedNumericDoubles(maxDoc), missingValue); + return select(FieldData.emptySortedNumericDoubles(), missingValue); } return new NumericDoubleValues() { @@ -677,32 +617,33 @@ public NumericDoubleValues select(final SortedNumericDoubleValues values, final double lastEmittedValue = missingValue; @Override - public double get(int rootDoc) { + public boolean advanceExact(int rootDoc) throws IOException { assert rootDocs.get(rootDoc) : "can only sort root documents"; assert rootDoc >= lastSeenRootDoc : "can only evaluate current and upcoming root docs"; if (rootDoc == lastSeenRootDoc) { - return lastEmittedValue; + return true; } - try { - final int prevRootDoc = rootDocs.prevSetBit(rootDoc - 1); - final int firstNestedDoc; - if (innerDocs.docID() > prevRootDoc) { - firstNestedDoc = innerDocs.docID(); - } else { - firstNestedDoc = innerDocs.advance(prevRootDoc + 1); - } - - lastSeenRootDoc = rootDoc; - lastEmittedValue = pick(values, missingValue, innerDocs, firstNestedDoc, rootDoc); - return lastEmittedValue; - } catch (IOException e) { - throw new RuntimeException(e); + final int prevRootDoc = rootDocs.prevSetBit(rootDoc - 1); + final int firstNestedDoc; + if (innerDocs.docID() > prevRootDoc) { + firstNestedDoc = innerDocs.docID(); + } else { + firstNestedDoc = innerDocs.advance(prevRootDoc + 1); } + + lastSeenRootDoc = rootDoc; + lastEmittedValue = pick(values, missingValue, innerDocs, firstNestedDoc, rootDoc); + return true; + } + + @Override + public double doubleValue() throws IOException { + return lastEmittedValue; } }; } - protected double pick(SortedNumericDoubleValues values, double missingValue, DocIdSetIterator docItr, int startDoc, int endDoc) { + protected double pick(SortedNumericDoubleValues values, double missingValue, DocIdSetIterator docItr, int startDoc, int endDoc) throws IOException { throw new IllegalArgumentException("Unsupported sort mode: " + this); } @@ -716,32 +657,41 @@ protected double pick(SortedNumericDoubleValues values, double missingValue, Doc public BinaryDocValues select(final SortedBinaryDocValues values, final BytesRef missingValue) { final BinaryDocValues singleton = FieldData.unwrapSingleton(values); if (singleton != null) { - final Bits docsWithField = FieldData.unwrapSingletonBits(values); - if (docsWithField == null) { - return singleton; - } else { - return new BinaryDocValues() { - @Override - public BytesRef get(int docID) { - final BytesRef value = singleton.get(docID); - if (value.length == 0 && docsWithField.get(docID) == false) { - return missingValue; - } - return value; - } - }; - } + return new AbstractBinaryDocValues() { + + private boolean hasValue; + + @Override + public boolean advanceExact(int target) throws IOException { + hasValue = singleton.advanceExact(target); + return true; + } + + @Override + public BytesRef binaryValue() throws IOException { + return hasValue ? singleton.binaryValue() : missingValue; + } + }; } else { - return new BinaryDocValues() { + return new AbstractBinaryDocValues() { + + private boolean hasValue; + @Override - public BytesRef get(int docID) { - return pick(values, missingValue, docID); + public boolean advanceExact(int target) throws IOException { + hasValue = values.advanceExact(target); + return true; + } + + @Override + public BytesRef binaryValue() throws IOException { + return hasValue ? pick(values) : missingValue; } }; } } - protected BytesRef pick(SortedBinaryDocValues values, BytesRef missingValue, int doc) { + protected BytesRef pick(SortedBinaryDocValues values) throws IOException { throw new IllegalArgumentException("Unsupported sort mode: " + this); } @@ -759,11 +709,11 @@ protected BytesRef pick(SortedBinaryDocValues values, BytesRef missingValue, int */ public BinaryDocValues select(final SortedBinaryDocValues values, final BytesRef missingValue, final BitSet rootDocs, final DocIdSetIterator innerDocs, int maxDoc) throws IOException { if (rootDocs == null || innerDocs == null) { - return select(FieldData.emptySortedBinary(maxDoc), missingValue); + return select(FieldData.emptySortedBinary(), missingValue); } final BinaryDocValues selectedValues = select(values, null); - return new BinaryDocValues() { + return new AbstractBinaryDocValues() { final BytesRefBuilder builder = new BytesRefBuilder(); @@ -771,36 +721,37 @@ public BinaryDocValues select(final SortedBinaryDocValues values, final BytesRef BytesRef lastEmittedValue = missingValue; @Override - public BytesRef get(int rootDoc) { + public boolean advanceExact(int rootDoc) throws IOException { assert rootDocs.get(rootDoc) : "can only sort root documents"; assert rootDoc >= lastSeenRootDoc : "can only evaluate current and upcoming root docs"; if (rootDoc == lastSeenRootDoc) { - return lastEmittedValue; + return true; } - try { - final int prevRootDoc = rootDocs.prevSetBit(rootDoc - 1); - final int firstNestedDoc; - if (innerDocs.docID() > prevRootDoc) { - firstNestedDoc = innerDocs.docID(); - } else { - firstNestedDoc = innerDocs.advance(prevRootDoc + 1); - } + final int prevRootDoc = rootDocs.prevSetBit(rootDoc - 1); + final int firstNestedDoc; + if (innerDocs.docID() > prevRootDoc) { + firstNestedDoc = innerDocs.docID(); + } else { + firstNestedDoc = innerDocs.advance(prevRootDoc + 1); + } - lastSeenRootDoc = rootDoc; - lastEmittedValue = pick(selectedValues, builder, innerDocs, firstNestedDoc, rootDoc); - if (lastEmittedValue == null) { - lastEmittedValue = missingValue; - } - return lastEmittedValue; - } catch (IOException e) { - throw new RuntimeException(e); + lastSeenRootDoc = rootDoc; + lastEmittedValue = pick(selectedValues, builder, innerDocs, firstNestedDoc, rootDoc); + if (lastEmittedValue == null) { + lastEmittedValue = missingValue; } + return true; + } + + @Override + public BytesRef binaryValue() throws IOException { + return lastEmittedValue; } }; } - protected BytesRef pick(BinaryDocValues values, BytesRefBuilder builder, DocIdSetIterator docItr, int startDoc, int endDoc) { + protected BytesRef pick(BinaryDocValues values, BytesRefBuilder builder, DocIdSetIterator docItr, int startDoc, int endDoc) throws IOException { throw new IllegalArgumentException("Unsupported sort mode: " + this); } @@ -810,7 +761,7 @@ protected BytesRef pick(BinaryDocValues values, BytesRefBuilder builder, DocIdSe * * Allowed Modes: MIN, MAX */ - public SortedDocValues select(final RandomAccessOrds values) { + public SortedDocValues select(final SortedSetDocValues values) { if (values.getValueCount() >= Integer.MAX_VALUE) { throw new UnsupportedOperationException("fields containing more than " + (Integer.MAX_VALUE-1) + " unique terms are unsupported"); } @@ -819,26 +770,37 @@ public SortedDocValues select(final RandomAccessOrds values) { if (singleton != null) { return singleton; } else { - return new SortedDocValues() { + return new AbstractSortedDocValues() { + @Override - public int getOrd(int docID) { - return pick(values, docID); + public boolean advanceExact(int target) throws IOException { + return values.advanceExact(target); } @Override - public BytesRef lookupOrd(int ord) { + public int ordValue() { + try { + return pick(values); + } catch (IOException e) { + // https://issues.apache.org/jira/browse/LUCENE-7767 + throw new RuntimeException(e); + } + } + + @Override + public BytesRef lookupOrd(int ord) throws IOException { return values.lookupOrd(ord); } @Override public int getValueCount() { - return (int)values.getValueCount(); + return (int) values.getValueCount(); } }; } } - protected int pick(RandomAccessOrds values, int doc) { + protected int pick(SortedSetDocValues values) throws IOException { throw new IllegalArgumentException("Unsupported sort mode: " + this); } @@ -853,19 +815,19 @@ protected int pick(RandomAccessOrds values, int doc) { * NOTE: Calling the returned instance on docs that are not root docs is illegal * The returned instance can only be evaluate the current and upcoming docs */ - public SortedDocValues select(final RandomAccessOrds values, final BitSet rootDocs, final DocIdSetIterator innerDocs) throws IOException { + public SortedDocValues select(final SortedSetDocValues values, final BitSet rootDocs, final DocIdSetIterator innerDocs) throws IOException { if (rootDocs == null || innerDocs == null) { return select(DocValues.emptySortedSet()); } final SortedDocValues selectedValues = select(values); - return new SortedDocValues() { + return new AbstractSortedDocValues() { int lastSeenRootDoc = 0; int lastEmittedOrd = -1; @Override - public BytesRef lookupOrd(int ord) { + public BytesRef lookupOrd(int ord) throws IOException { return selectedValues.lookupOrd(ord); } @@ -875,32 +837,34 @@ public int getValueCount() { } @Override - public int getOrd(int rootDoc) { + public boolean advanceExact(int rootDoc) throws IOException { assert rootDocs.get(rootDoc) : "can only sort root documents"; assert rootDoc >= lastSeenRootDoc : "can only evaluate current and upcoming root docs"; if (rootDoc == lastSeenRootDoc) { - return lastEmittedOrd; + return true; } - try { - final int prevRootDoc = rootDocs.prevSetBit(rootDoc - 1); - final int firstNestedDoc; - if (innerDocs.docID() > prevRootDoc) { - firstNestedDoc = innerDocs.docID(); - } else { - firstNestedDoc = innerDocs.advance(prevRootDoc + 1); - } - - lastSeenRootDoc = rootDoc; - return lastEmittedOrd = pick(selectedValues, innerDocs, firstNestedDoc, rootDoc); - } catch (IOException e) { - throw new RuntimeException(e); + final int prevRootDoc = rootDocs.prevSetBit(rootDoc - 1); + final int firstNestedDoc; + if (innerDocs.docID() > prevRootDoc) { + firstNestedDoc = innerDocs.docID(); + } else { + firstNestedDoc = innerDocs.advance(prevRootDoc + 1); } + + lastSeenRootDoc = rootDoc; + lastEmittedOrd = pick(selectedValues, innerDocs, firstNestedDoc, rootDoc); + return true; + } + + @Override + public int ordValue() { + return lastEmittedOrd; } }; } - protected int pick(SortedDocValues values, DocIdSetIterator docItr, int startDoc, int endDoc) { + protected int pick(SortedDocValues values, DocIdSetIterator docItr, int startDoc, int endDoc) throws IOException { throw new IllegalArgumentException("Unsupported sort mode: " + this); } @@ -913,24 +877,32 @@ protected int pick(SortedDocValues values, DocIdSetIterator docItr, int startDoc */ public NumericDoubleValues select(final UnsortedNumericDoubleValues values, final double missingValue) { return new NumericDoubleValues() { + private boolean hasValue; + + @Override + public boolean advanceExact(int doc) throws IOException { + hasValue = values.advanceExact(doc); + return true; + } @Override - public double get(int docID) { - return pick(values, missingValue, docID); + public double doubleValue() throws IOException { + return hasValue ? pick(values) : missingValue; } }; } - protected double pick(UnsortedNumericDoubleValues values, final double missingValue, int doc) { + protected double pick(UnsortedNumericDoubleValues values) throws IOException { throw new IllegalArgumentException("Unsupported sort mode: " + this); } /** * Interface allowing custom value generators to be used in MultiValueMode. */ + // TODO: why do we need it??? public interface UnsortedNumericDoubleValues { - int count(); - void setDocument(int docId); - double valueAt(int index); + boolean advanceExact(int doc) throws IOException; + int docValueCount() throws IOException; + double nextValue() throws IOException; } @Override From 8cadffcff63e4cf5d245f78bef5ef6009b682ad0 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Wed, 5 Apr 2017 14:42:51 +0200 Subject: [PATCH 08/87] Goodbye query normalization. --- .../common/lucene/all/AllTermQuery.java | 16 +++------------- .../function/FiltersFunctionScoreQuery.java | 16 +++------------- .../search/function/FunctionScoreQuery.java | 16 +++------------- .../elasticsearch/indices/IndicesQueryCache.java | 10 ---------- .../indices/IndicesRequestCache.java | 4 ++-- .../search/profile/query/ProfileWeight.java | 10 ---------- .../elasticsearch/percolator/PercolateQuery.java | 16 +++------------- 7 files changed, 14 insertions(+), 74 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/common/lucene/all/AllTermQuery.java b/core/src/main/java/org/elasticsearch/common/lucene/all/AllTermQuery.java index 5307a417e107d..7df146a11c231 100644 --- a/core/src/main/java/org/elasticsearch/common/lucene/all/AllTermQuery.java +++ b/core/src/main/java/org/elasticsearch/common/lucene/all/AllTermQuery.java @@ -105,27 +105,17 @@ public Query rewrite(IndexReader reader) throws IOException { } @Override - public Weight createWeight(IndexSearcher searcher, boolean needsScores) throws IOException { + public Weight createWeight(IndexSearcher searcher, boolean needsScores, float boost) throws IOException { if (needsScores == false) { - return new TermQuery(term).createWeight(searcher, needsScores); + return new TermQuery(term).createWeight(searcher, needsScores, boost); } final TermContext termStates = TermContext.build(searcher.getTopReaderContext(), term); final CollectionStatistics collectionStats = searcher.collectionStatistics(term.field()); final TermStatistics termStats = searcher.termStatistics(term, termStates); final Similarity similarity = searcher.getSimilarity(needsScores); - final SimWeight stats = similarity.computeWeight(collectionStats, termStats); + final SimWeight stats = similarity.computeWeight(boost, collectionStats, termStats); return new Weight(this) { - @Override - public float getValueForNormalization() throws IOException { - return stats.getValueForNormalization(); - } - - @Override - public void normalize(float norm, float topLevelBoost) { - stats.normalize(norm, topLevelBoost); - } - @Override public void extractTerms(Set terms) { terms.add(term); diff --git a/core/src/main/java/org/elasticsearch/common/lucene/search/function/FiltersFunctionScoreQuery.java b/core/src/main/java/org/elasticsearch/common/lucene/search/function/FiltersFunctionScoreQuery.java index fe12622748ef5..801f399a669d0 100644 --- a/core/src/main/java/org/elasticsearch/common/lucene/search/function/FiltersFunctionScoreQuery.java +++ b/core/src/main/java/org/elasticsearch/common/lucene/search/function/FiltersFunctionScoreQuery.java @@ -135,9 +135,9 @@ public Query rewrite(IndexReader reader) throws IOException { } @Override - public Weight createWeight(IndexSearcher searcher, boolean needsScores) throws IOException { + public Weight createWeight(IndexSearcher searcher, boolean needsScores, float boost) throws IOException { if (needsScores == false && minScore == null) { - return subQuery.createWeight(searcher, needsScores); + return subQuery.createWeight(searcher, needsScores, boost); } boolean subQueryNeedsScores = combineFunction != CombineFunction.REPLACE; @@ -146,7 +146,7 @@ public Weight createWeight(IndexSearcher searcher, boolean needsScores) throws I subQueryNeedsScores |= filterFunctions[i].function.needsScores(); filterWeights[i] = searcher.createNormalizedWeight(filterFunctions[i].filter, false); } - Weight subQueryWeight = subQuery.createWeight(searcher, subQueryNeedsScores); + Weight subQueryWeight = subQuery.createWeight(searcher, subQueryNeedsScores, boost); return new CustomBoostFactorWeight(this, subQueryWeight, filterWeights, subQueryNeedsScores); } @@ -168,16 +168,6 @@ public void extractTerms(Set terms) { subQueryWeight.extractTerms(terms); } - @Override - public float getValueForNormalization() throws IOException { - return subQueryWeight.getValueForNormalization(); - } - - @Override - public void normalize(float norm, float boost) { - subQueryWeight.normalize(norm, boost); - } - private FiltersFunctionFactorScorer functionScorer(LeafReaderContext context) throws IOException { Scorer subQueryScorer = subQueryWeight.scorer(context); if (subQueryScorer == null) { diff --git a/core/src/main/java/org/elasticsearch/common/lucene/search/function/FunctionScoreQuery.java b/core/src/main/java/org/elasticsearch/common/lucene/search/function/FunctionScoreQuery.java index 61de1ab303f24..95fc58dce59b4 100644 --- a/core/src/main/java/org/elasticsearch/common/lucene/search/function/FunctionScoreQuery.java +++ b/core/src/main/java/org/elasticsearch/common/lucene/search/function/FunctionScoreQuery.java @@ -91,16 +91,16 @@ public Query rewrite(IndexReader reader) throws IOException { } @Override - public Weight createWeight(IndexSearcher searcher, boolean needsScores) throws IOException { + public Weight createWeight(IndexSearcher searcher, boolean needsScores, float boost) throws IOException { if (needsScores == false && minScore == null) { - return subQuery.createWeight(searcher, needsScores); + return subQuery.createWeight(searcher, needsScores, boost); } boolean subQueryNeedsScores = combineFunction != CombineFunction.REPLACE // if we don't replace we need the original score || function == null // when the function is null, we just multiply the score, so we need it || function.needsScores(); // some scripts can replace with a script that returns eg. 1/_score - Weight subQueryWeight = subQuery.createWeight(searcher, subQueryNeedsScores); + Weight subQueryWeight = subQuery.createWeight(searcher, subQueryNeedsScores, boost); return new CustomBoostFactorWeight(this, subQueryWeight, subQueryNeedsScores); } @@ -120,16 +120,6 @@ public void extractTerms(Set terms) { subQueryWeight.extractTerms(terms); } - @Override - public float getValueForNormalization() throws IOException { - return subQueryWeight.getValueForNormalization(); - } - - @Override - public void normalize(float norm, float boost) { - subQueryWeight.normalize(norm, boost); - } - private FunctionFactorScorer functionScorer(LeafReaderContext context) throws IOException { Scorer subQueryScorer = subQueryWeight.scorer(context); if (subQueryScorer == null) { diff --git a/core/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java b/core/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java index f33818ca32b3f..3fda3d3f806d6 100644 --- a/core/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java +++ b/core/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java @@ -139,16 +139,6 @@ public Explanation explain(LeafReaderContext context, int doc) throws IOExceptio return in.explain(context, doc); } - @Override - public float getValueForNormalization() throws IOException { - return in.getValueForNormalization(); - } - - @Override - public void normalize(float norm, float topLevelBoost) { - in.normalize(norm, topLevelBoost); - } - @Override public Scorer scorer(LeafReaderContext context) throws IOException { shardKeyMap.add(context.reader()); diff --git a/core/src/main/java/org/elasticsearch/indices/IndicesRequestCache.java b/core/src/main/java/org/elasticsearch/indices/IndicesRequestCache.java index 48560f102d263..d941b097a6010 100644 --- a/core/src/main/java/org/elasticsearch/indices/IndicesRequestCache.java +++ b/core/src/main/java/org/elasticsearch/indices/IndicesRequestCache.java @@ -245,7 +245,7 @@ public int hashCode() { } } - private class CleanupKey implements IndexReader.ReaderClosedListener { + private class CleanupKey implements IndexReader.ClosedListener { final CacheEntity entity; final long readerVersion; // use the reader version to now keep a reference to a "short" lived reader until its reaped @@ -255,7 +255,7 @@ private CleanupKey(CacheEntity entity, long readerVersion) { } @Override - public void onClose(IndexReader reader) { + public void onClose(IndexReader.CacheKey cacheKey) { Boolean remove = registeredClosedListeners.remove(this); if (remove != null) { keysToClean.add(this); diff --git a/core/src/main/java/org/elasticsearch/search/profile/query/ProfileWeight.java b/core/src/main/java/org/elasticsearch/search/profile/query/ProfileWeight.java index 9ca33f8454253..716f68da86f89 100644 --- a/core/src/main/java/org/elasticsearch/search/profile/query/ProfileWeight.java +++ b/core/src/main/java/org/elasticsearch/search/profile/query/ProfileWeight.java @@ -79,16 +79,6 @@ public Explanation explain(LeafReaderContext context, int doc) throws IOExceptio return subQueryWeight.explain(context, doc); } - @Override - public float getValueForNormalization() throws IOException { - return subQueryWeight.getValueForNormalization(); - } - - @Override - public void normalize(float norm, float topLevelBoost) { - subQueryWeight.normalize(norm, topLevelBoost); - } - @Override public void extractTerms(Set set) { subQueryWeight.extractTerms(set); diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQuery.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQuery.java index f5978a2dbd1c7..17cdfc68c83f2 100644 --- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQuery.java +++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQuery.java @@ -74,9 +74,9 @@ public Query rewrite(IndexReader reader) throws IOException { } @Override - public Weight createWeight(IndexSearcher searcher, boolean needsScores) throws IOException { - final Weight verifiedMatchesWeight = verifiedMatchesQuery.createWeight(searcher, false); - final Weight candidateMatchesWeight = candidateMatchesQuery.createWeight(searcher, false); + public Weight createWeight(IndexSearcher searcher, boolean needsScores, float boost) throws IOException { + final Weight verifiedMatchesWeight = verifiedMatchesQuery.createWeight(searcher, false, boost); + final Weight candidateMatchesWeight = candidateMatchesQuery.createWeight(searcher, false, boost); return new Weight(this) { @Override public void extractTerms(Set set) { @@ -104,16 +104,6 @@ public Explanation explain(LeafReaderContext leafReaderContext, int docId) throw return Explanation.noMatch("PercolateQuery"); } - @Override - public float getValueForNormalization() throws IOException { - return candidateMatchesWeight.getValueForNormalization(); - } - - @Override - public void normalize(float v, float v1) { - candidateMatchesWeight.normalize(v, v1); - } - @Override public Scorer scorer(LeafReaderContext leafReaderContext) throws IOException { final Scorer approximation = candidateMatchesWeight.scorer(leafReaderContext); From 3602f656d6b577c516513f505fde89a6584d9f53 Mon Sep 17 00:00:00 2001 From: Colin Goodheart-Smithe Date: Wed, 5 Apr 2017 13:27:04 +0100 Subject: [PATCH 09/87] Fixes compile errors in ValuesSource --- .../AbstractSortingNumericDocValues.java | 54 +++++++++ .../fielddata/AtomicOrdinalsFieldData.java | 4 +- .../fielddata/SortingNumericDocValues.java | 9 +- .../fielddata/SortingNumericDoubleValues.java | 9 +- .../aggregations/support/ValuesSource.java | 114 ++++++++++-------- 5 files changed, 132 insertions(+), 58 deletions(-) create mode 100644 core/src/main/java/org/elasticsearch/index/fielddata/AbstractSortingNumericDocValues.java diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/AbstractSortingNumericDocValues.java b/core/src/main/java/org/elasticsearch/index/fielddata/AbstractSortingNumericDocValues.java new file mode 100644 index 0000000000000..ad4b8c08cc971 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/index/fielddata/AbstractSortingNumericDocValues.java @@ -0,0 +1,54 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.fielddata; + +import org.apache.lucene.search.DocIdSetIterator; + +import java.io.IOException; + +/** + * Base implementation that throws an {@link IOException} for the + * {@link DocIdSetIterator} APIs. This impl is safe to use for sorting and + * aggregations, which only use {@link #advanceExact(int)} and + * {@link #docValueCount()} and {@link #nextValue()}. + */ +public abstract class AbstractSortingNumericDocValues extends SortingNumericDocValues { + + @Override + public int docID() { + throw new UnsupportedOperationException(); + } + + @Override + public int nextDoc() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public int advance(int target) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public long cost() { + throw new UnsupportedOperationException(); + } + +} diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/AtomicOrdinalsFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/AtomicOrdinalsFieldData.java index 8d1fd310571c5..14ed16ecb3ec9 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/AtomicOrdinalsFieldData.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/AtomicOrdinalsFieldData.java @@ -19,7 +19,7 @@ package org.elasticsearch.index.fielddata; -import org.apache.lucene.index.RandomAccessOrds; +import org.apache.lucene.index.SortedSetDocValues; /** * Specialization of {@link AtomicFieldData} for data that is indexed with @@ -30,6 +30,6 @@ public interface AtomicOrdinalsFieldData extends AtomicFieldData { /** * Return the ordinals values for the current atomic reader. */ - RandomAccessOrds getOrdinalsValues(); + SortedSetDocValues getOrdinalsValues(); } diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/SortingNumericDocValues.java b/core/src/main/java/org/elasticsearch/index/fielddata/SortingNumericDocValues.java index 7c1d1fff2aed6..082892dce2ab7 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/SortingNumericDocValues.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/SortingNumericDocValues.java @@ -31,10 +31,12 @@ public abstract class SortingNumericDocValues extends SortedNumericDocValues { private int count; protected long[] values; + protected int valuesCursor; private final Sorter sorter; protected SortingNumericDocValues() { values = new long[1]; + valuesCursor = 0; sorter = new InPlaceMergeSorter() { @Override @@ -58,6 +60,7 @@ protected int compare(int i, int j) { protected final void resize(int newSize) { count = newSize; values = ArrayUtil.grow(values, count); + valuesCursor = 0; } /** @@ -69,12 +72,12 @@ protected final void sort() { } @Override - public final int count() { + public final int docValueCount() { return count; } @Override - public final long valueAt(int index) { - return values[index]; + public final long nextValue() { + return values[valuesCursor++]; } } diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/SortingNumericDoubleValues.java b/core/src/main/java/org/elasticsearch/index/fielddata/SortingNumericDoubleValues.java index 674a86719ec41..c3aac671555e8 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/SortingNumericDoubleValues.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/SortingNumericDoubleValues.java @@ -29,11 +29,13 @@ public abstract class SortingNumericDoubleValues extends SortedNumericDoubleValues { private int count; + private int valuesCursor; protected double[] values; private final Sorter sorter; protected SortingNumericDoubleValues() { values = new double[1]; + valuesCursor = 0; sorter = new InPlaceMergeSorter() { @Override @@ -57,6 +59,7 @@ protected int compare(int i, int j) { protected final void resize(int newSize) { count = newSize; values = ArrayUtil.grow(values, count); + valuesCursor = 0; } /** @@ -68,12 +71,12 @@ protected final void sort() { } @Override - public final int count() { + public final int docValueCount() { return count; } @Override - public final double valueAt(int index) { - return values[index]; + public final double nextValue() { + return values[valuesCursor++]; } } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSource.java b/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSource.java index f3de52ae6534c..672447720ed9f 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSource.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSource.java @@ -22,14 +22,15 @@ import org.apache.lucene.index.DocValues; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.RandomAccessOrds; import org.apache.lucene.index.SortedDocValues; import org.apache.lucene.index.SortedNumericDocValues; +import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Scorer; import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.lucene.ScorerAware; +import org.elasticsearch.index.fielddata.AbstractSortingNumericDocValues; import org.elasticsearch.index.fielddata.AtomicOrdinalsFieldData; import org.elasticsearch.index.fielddata.AtomicParentChildFieldData; import org.elasticsearch.index.fielddata.IndexFieldData; @@ -41,7 +42,6 @@ import org.elasticsearch.index.fielddata.SortedBinaryDocValues; import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; import org.elasticsearch.index.fielddata.SortingBinaryDocValues; -import org.elasticsearch.index.fielddata.SortingNumericDocValues; import org.elasticsearch.index.fielddata.SortingNumericDoubleValues; import org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData; import org.elasticsearch.script.LeafSearchScript; @@ -84,25 +84,25 @@ public abstract static class WithOrdinals extends Bytes { public static final WithOrdinals EMPTY = new WithOrdinals() { @Override - public RandomAccessOrds ordinalsValues(LeafReaderContext context) { + public SortedSetDocValues ordinalsValues(LeafReaderContext context) { return DocValues.emptySortedSet(); } @Override - public RandomAccessOrds globalOrdinalsValues(LeafReaderContext context) { + public SortedSetDocValues globalOrdinalsValues(LeafReaderContext context) { return DocValues.emptySortedSet(); } @Override public SortedBinaryDocValues bytesValues(LeafReaderContext context) throws IOException { - return org.elasticsearch.index.fielddata.FieldData.emptySortedBinary(context.reader().maxDoc()); + return org.elasticsearch.index.fielddata.FieldData.emptySortedBinary(); } }; @Override public Bits docsWithValue(LeafReaderContext context) { - final RandomAccessOrds ordinals = ordinalsValues(context); + final SortedSetDocValues ordinals = ordinalsValues(context); if (DocValues.unwrapSingleton(ordinals) != null) { return DocValues.docsWithValue(DocValues.unwrapSingleton(ordinals), context.reader().maxDoc()); } else { @@ -110,9 +110,9 @@ public Bits docsWithValue(LeafReaderContext context) { } } - public abstract RandomAccessOrds ordinalsValues(LeafReaderContext context); + public abstract SortedSetDocValues ordinalsValues(LeafReaderContext context); - public abstract RandomAccessOrds globalOrdinalsValues(LeafReaderContext context); + public abstract SortedSetDocValues globalOrdinalsValues(LeafReaderContext context); public long globalMaxOrd(IndexSearcher indexSearcher) { IndexReader indexReader = indexSearcher.getIndexReader(); @@ -120,7 +120,7 @@ public long globalMaxOrd(IndexSearcher indexSearcher) { return 0; } else { LeafReaderContext atomicReaderContext = indexReader.leaves().get(0); - RandomAccessOrds values = globalOrdinalsValues(atomicReaderContext); + SortedSetDocValues values = globalOrdinalsValues(atomicReaderContext); return values.getValueCount(); } } @@ -140,13 +140,13 @@ public SortedBinaryDocValues bytesValues(LeafReaderContext context) { } @Override - public RandomAccessOrds ordinalsValues(LeafReaderContext context) { + public SortedSetDocValues ordinalsValues(LeafReaderContext context) { final AtomicOrdinalsFieldData atomicFieldData = indexFieldData.load(context); return atomicFieldData.getOrdinalsValues(); } @Override - public RandomAccessOrds globalOrdinalsValues(LeafReaderContext context) { + public SortedSetDocValues globalOrdinalsValues(LeafReaderContext context) { final IndexOrdinalsFieldData global = indexFieldData.loadGlobal((DirectoryReader)context.parent.reader()); final AtomicOrdinalsFieldData atomicFieldData = global.load(context); return atomicFieldData.getOrdinalsValues(); @@ -240,12 +240,12 @@ public SortedNumericDocValues longValues(LeafReaderContext context) { @Override public SortedNumericDoubleValues doubleValues(LeafReaderContext context) throws IOException { - return org.elasticsearch.index.fielddata.FieldData.emptySortedNumericDoubles(context.reader().maxDoc()); + return org.elasticsearch.index.fielddata.FieldData.emptySortedNumericDoubles(); } @Override public SortedBinaryDocValues bytesValues(LeafReaderContext context) throws IOException { - return org.elasticsearch.index.fielddata.FieldData.emptySortedBinary(context.reader().maxDoc()); + return org.elasticsearch.index.fielddata.FieldData.emptySortedBinary(); } }; @@ -313,7 +313,7 @@ public SortedNumericDoubleValues doubleValues(LeafReaderContext context) throws return new DoubleValues(delegate.doubleValues(context), script.getLeafSearchScript(context)); } - static class LongValues extends SortingNumericDocValues implements ScorerAware { + static class LongValues extends AbstractSortingNumericDocValues implements ScorerAware { private final SortedNumericDocValues longValues; private final LeafSearchScript script; @@ -324,20 +324,24 @@ static class LongValues extends SortingNumericDocValues implements ScorerAware { } @Override - public void setDocument(int doc) { - longValues.setDocument(doc); - resize(longValues.count()); - script.setDocument(doc); - for (int i = 0; i < count(); ++i) { - script.setNextAggregationValue(longValues.valueAt(i)); - values[i] = script.runAsLong(); - } - sort(); + public void setScorer(Scorer scorer) { + script.setScorer(scorer); } @Override - public void setScorer(Scorer scorer) { - script.setScorer(scorer); + public boolean advanceExact(int target) throws IOException { + if (longValues.advanceExact(target)) { + resize(longValues.docValueCount()); + script.setDocument(target); + for (int i = 0; i < docValueCount(); ++i) { + script.setNextAggregationValue(longValues.nextValue()); + values[i] = script.runAsLong(); + } + sort(); + return true; + } else { + return false; + } } } @@ -352,20 +356,24 @@ static class DoubleValues extends SortingNumericDoubleValues implements ScorerAw } @Override - public void setDocument(int doc) { - doubleValues.setDocument(doc); - resize(doubleValues.count()); - script.setDocument(doc); - for (int i = 0; i < count(); ++i) { - script.setNextAggregationValue(doubleValues.valueAt(i)); - values[i] = script.runAsDouble(); - } - sort(); + public void setScorer(Scorer scorer) { + script.setScorer(scorer); } @Override - public void setScorer(Scorer scorer) { - script.setScorer(scorer); + public boolean advanceExact(int target) throws IOException { + if (doubleValues.advanceExact(target)) { + resize(doubleValues.docValueCount()); + script.setDocument(target); + for (int i = 0; i < docValueCount(); ++i) { + script.setNextAggregationValue(doubleValues.nextValue()); + values[i] = script.runAsLong(); + } + sort(); + return true; + } else { + return false; + } } } } @@ -468,21 +476,27 @@ static class BytesValues extends SortingBinaryDocValues implements ScorerAware { } @Override - public void setDocument(int docId) { - bytesValues.setDocument(docId); - count = bytesValues.count(); - grow(); - for (int i = 0; i < count; ++i) { - final BytesRef value = bytesValues.valueAt(i); - script.setNextAggregationValue(value.utf8ToString()); - values[i].copyChars(script.run().toString()); - } - sort(); + public void setScorer(Scorer scorer) { + script.setScorer(scorer); } @Override - public void setScorer(Scorer scorer) { - script.setScorer(scorer); + public boolean advanceExact(int doc) throws IOException { + if (bytesValues.advanceExact(doc)) { + count = bytesValues.docValueCount(); + grow(); + for (int i = 0; i < count; ++i) { + final BytesRef value = bytesValues.nextValue(); + script.setNextAggregationValue(value.utf8ToString()); + values[i].copyChars(script.run().toString()); + } + sort(); + return true; + } else { + count = 0; + grow(); + return false; + } } } } @@ -493,12 +507,12 @@ public abstract static class GeoPoint extends ValuesSource { @Override public MultiGeoPointValues geoPointValues(LeafReaderContext context) { - return org.elasticsearch.index.fielddata.FieldData.emptyMultiGeoPoints(context.reader().maxDoc()); + return org.elasticsearch.index.fielddata.FieldData.emptyMultiGeoPoints(); } @Override public SortedBinaryDocValues bytesValues(LeafReaderContext context) throws IOException { - return org.elasticsearch.index.fielddata.FieldData.emptySortedBinary(context.reader().maxDoc()); + return org.elasticsearch.index.fielddata.FieldData.emptySortedBinary(); } }; From f2a52dbfe3cc2fb7f1d48febd819b8c072f33df2 Mon Sep 17 00:00:00 2001 From: Colin Goodheart-Smithe Date: Wed, 5 Apr 2017 14:01:04 +0100 Subject: [PATCH 10/87] Fiexes compile errors in CardinalityAggregator --- .../cardinality/CardinalityAggregator.java | 73 ++++++++++--------- 1 file changed, 38 insertions(+), 35 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/CardinalityAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/CardinalityAggregator.java index 7d5db460ae6b0..b982590e5234a 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/CardinalityAggregator.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/CardinalityAggregator.java @@ -22,8 +22,8 @@ import com.carrotsearch.hppc.BitMixer; import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.RandomAccessOrds; import org.apache.lucene.index.SortedNumericDocValues; +import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.FixedBitSet; @@ -88,7 +88,7 @@ private Collector pickCollector(LeafReaderContext ctx) throws IOException { if (valuesSource instanceof ValuesSource.Bytes.WithOrdinals) { ValuesSource.Bytes.WithOrdinals source = (ValuesSource.Bytes.WithOrdinals) valuesSource; - final RandomAccessOrds ordinalValues = source.ordinalsValues(ctx); + final SortedSetDocValues ordinalValues = source.ordinalsValues(ctx); final long maxOrd = ordinalValues.getValueCount(); if (maxOrd == 0) { return new EmptyCollector(); @@ -114,7 +114,7 @@ public LeafBucketCollector getLeafCollector(LeafReaderContext ctx, return collector; } - private void postCollectLastCollector() { + private void postCollectLastCollector() throws IOException { if (collector != null) { try { collector.postCollect(); @@ -126,7 +126,7 @@ private void postCollectLastCollector() { } @Override - protected void doPostCollection() { + protected void doPostCollection() throws IOException { postCollectLastCollector(); } @@ -159,7 +159,7 @@ protected void doClose() { private abstract static class Collector extends LeafBucketCollector implements Releasable { - public abstract void postCollect(); + public abstract void postCollect() throws IOException; } @@ -192,11 +192,12 @@ private static class DirectCollector extends Collector { } @Override - public void collect(int doc, long bucketOrd) { - hashes.setDocument(doc); - final int valueCount = hashes.count(); - for (int i = 0; i < valueCount; ++i) { - counts.collect(bucketOrd, hashes.valueAt(i)); + public void collect(int doc, long bucketOrd) throws IOException { + if (hashes.advanceExact(doc)) { + final int valueCount = hashes.count(); + for (int i = 0; i < valueCount; ++i) { + counts.collect(bucketOrd, hashes.nextValue()); + } } } @@ -224,12 +225,13 @@ public static long memoryOverhead(long maxOrd) { } private final BigArrays bigArrays; - private final RandomAccessOrds values; + private final SortedSetDocValues values; private final int maxOrd; private final HyperLogLogPlusPlus counts; private ObjectArray visitedOrds; - OrdinalsCollector(HyperLogLogPlusPlus counts, RandomAccessOrds values, BigArrays bigArrays) { + OrdinalsCollector(HyperLogLogPlusPlus counts, SortedSetDocValues values, + BigArrays bigArrays) { if (values.getValueCount() > Integer.MAX_VALUE) { throw new IllegalArgumentException(); } @@ -241,22 +243,23 @@ public static long memoryOverhead(long maxOrd) { } @Override - public void collect(int doc, long bucketOrd) { + public void collect(int doc, long bucketOrd) throws IOException { visitedOrds = bigArrays.grow(visitedOrds, bucketOrd + 1); FixedBitSet bits = visitedOrds.get(bucketOrd); if (bits == null) { bits = new FixedBitSet(maxOrd); visitedOrds.set(bucketOrd, bits); } - values.setDocument(doc); - final int valueCount = values.cardinality(); - for (int i = 0; i < valueCount; ++i) { - bits.set((int) values.ordAt(i)); + if (values.advanceExact(doc)) { + final long valueCount = values.getValueCount(); + for (int i = 0; i < valueCount; ++i) { + bits.set((int) values.nextOrd()); + } } } @Override - public void postCollect() { + public void postCollect() throws IOException { final FixedBitSet allVisitedOrds = new FixedBitSet(maxOrd); for (long bucket = visitedOrds.size() - 1; bucket >= 0; --bucket) { final FixedBitSet bits = visitedOrds.get(bucket); @@ -296,11 +299,11 @@ public void close() { */ abstract static class MurmurHash3Values { - public abstract void setDocument(int docId); + public abstract boolean advanceExact(int docId) throws IOException; public abstract int count(); - public abstract long valueAt(int index); + public abstract long nextValue() throws IOException; /** * Return a {@link MurmurHash3Values} instance that computes hashes on the fly for each double value. @@ -332,18 +335,18 @@ private static class Long extends MurmurHash3Values { } @Override - public void setDocument(int docId) { - values.setDocument(docId); + public boolean advanceExact(int docId) throws IOException { + return values.advanceExact(docId); } @Override public int count() { - return values.count(); + return values.docValueCount(); } @Override - public long valueAt(int index) { - return BitMixer.mix64(values.valueAt(index)); + public long nextValue() throws IOException { + return BitMixer.mix64(values.nextValue()); } } @@ -356,18 +359,18 @@ private static class Double extends MurmurHash3Values { } @Override - public void setDocument(int docId) { - values.setDocument(docId); + public boolean advanceExact(int docId) throws IOException { + return values.advanceExact(docId); } @Override public int count() { - return values.count(); + return values.docValueCount(); } @Override - public long valueAt(int index) { - return BitMixer.mix64(java.lang.Double.doubleToLongBits(values.valueAt(index))); + public long nextValue() throws IOException { + return BitMixer.mix64(java.lang.Double.doubleToLongBits(values.nextValue())); } } @@ -382,18 +385,18 @@ private static class Bytes extends MurmurHash3Values { } @Override - public void setDocument(int docId) { - values.setDocument(docId); + public boolean advanceExact(int docId) throws IOException { + return values.advanceExact(docId); } @Override public int count() { - return values.count(); + return values.docValueCount(); } @Override - public long valueAt(int index) { - final BytesRef bytes = values.valueAt(index); + public long nextValue() throws IOException { + final BytesRef bytes = values.nextValue(); org.elasticsearch.common.hash.MurmurHash3.hash128(bytes.bytes, bytes.offset, bytes.length, 0, hash); return hash.h1; } From 9e17d4f70de9091b2f1d411c4176f5838a7ddee1 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Wed, 5 Apr 2017 15:07:05 +0200 Subject: [PATCH 11/87] Fix more compile errors. --- .../ordinals/GlobalOrdinalMapping.java | 53 +++++++++++++------ .../ordinals/GlobalOrdinalsBuilder.java | 10 ++-- .../InternalGlobalOrdinalsIndexFieldData.java | 12 ++--- .../AbstractAtomicOrdinalsFieldData.java | 12 ++--- .../plain/DocValuesIndexFieldData.java | 5 +- .../SortedSetDVBytesAtomicFieldData.java | 9 ++-- .../SortedSetDVOrdinalsIndexFieldData.java | 8 +-- .../index/mapper/CustomDocValuesField.java | 5 -- .../index/mapper/DocumentMapper.java | 2 +- .../index/mapper/FieldMapper.java | 14 ----- .../index/mapper/GeoShapeFieldMapper.java | 5 -- .../index/mapper/IpFieldMapper.java | 46 ++++++++++++---- .../index/mapper/ScaledFloatFieldMapper.java | 24 +++++---- 13 files changed, 117 insertions(+), 88 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/GlobalOrdinalMapping.java b/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/GlobalOrdinalMapping.java index 293ddf430dfaf..65f70e85d50e4 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/GlobalOrdinalMapping.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/GlobalOrdinalMapping.java @@ -20,22 +20,23 @@ package org.elasticsearch.index.fielddata.ordinals; import org.apache.lucene.index.MultiDocValues.OrdinalMap; -import org.apache.lucene.index.RandomAccessOrds; +import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.LongValues; -import org.elasticsearch.index.fielddata.AbstractRandomAccessOrds; + +import java.io.IOException; /** * A {@link RandomAccessOrds} implementation that returns ordinals that are global. */ -public class GlobalOrdinalMapping extends AbstractRandomAccessOrds { +public class GlobalOrdinalMapping extends SortedSetDocValues { - private final RandomAccessOrds values; + private final SortedSetDocValues values; private final OrdinalMap ordinalMap; private final LongValues mapping; - private final RandomAccessOrds[] bytesValues; + private final SortedSetDocValues[] bytesValues; - GlobalOrdinalMapping(OrdinalMap ordinalMap, RandomAccessOrds[] bytesValues, int segmentIndex) { + GlobalOrdinalMapping(OrdinalMap ordinalMap, SortedSetDocValues[] bytesValues, int segmentIndex) { super(); this.values = bytesValues[segmentIndex]; this.bytesValues = bytesValues; @@ -53,25 +54,45 @@ public final long getGlobalOrd(long segmentOrd) { } @Override - public long ordAt(int index) { - return getGlobalOrd(values.ordAt(index)); - } - - @Override - public void doSetDocument(int docId) { - values.setDocument(docId); + public boolean advanceExact(int target) throws IOException { + return values.advanceExact(target); } @Override - public int cardinality() { - return values.cardinality(); + public long nextOrd() throws IOException { + long segmentOrd = values.nextOrd(); + if (segmentOrd == SortedSetDocValues.NO_MORE_ORDS) { + return SortedSetDocValues.NO_MORE_ORDS; + } else { + return getGlobalOrd(segmentOrd); + } } @Override - public BytesRef lookupOrd(long globalOrd) { + public BytesRef lookupOrd(long globalOrd) throws IOException { final long segmentOrd = ordinalMap.getFirstSegmentOrd(globalOrd); int readerIndex = ordinalMap.getFirstSegmentNumber(globalOrd); return bytesValues[readerIndex].lookupOrd(segmentOrd); } + @Override + public int docID() { + return values.docID(); + } + + @Override + public int nextDoc() throws IOException { + return values.nextDoc(); + } + + @Override + public int advance(int target) throws IOException { + return values.advance(target); + } + + @Override + public long cost() { + return values.cost(); + } + } diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/GlobalOrdinalsBuilder.java b/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/GlobalOrdinalsBuilder.java index 7b3b0646891f5..49140968ca7df 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/GlobalOrdinalsBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/GlobalOrdinalsBuilder.java @@ -23,7 +23,7 @@ import org.apache.lucene.index.DocValues; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.MultiDocValues.OrdinalMap; -import org.apache.lucene.index.RandomAccessOrds; +import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.util.Accountable; import org.apache.lucene.util.packed.PackedInts; import org.elasticsearch.common.breaker.CircuitBreaker; @@ -52,12 +52,12 @@ public enum GlobalOrdinalsBuilder { */ public static IndexOrdinalsFieldData build(final IndexReader indexReader, IndexOrdinalsFieldData indexFieldData, IndexSettings indexSettings, CircuitBreakerService breakerService, Logger logger, - Function> scriptFunction) throws IOException { + Function> scriptFunction) throws IOException { assert indexReader.leaves().size() > 1; long startTimeNS = System.nanoTime(); final AtomicOrdinalsFieldData[] atomicFD = new AtomicOrdinalsFieldData[indexReader.leaves().size()]; - final RandomAccessOrds[] subs = new RandomAccessOrds[indexReader.leaves().size()]; + final SortedSetDocValues[] subs = new SortedSetDocValues[indexReader.leaves().size()]; for (int i = 0; i < indexReader.leaves().size(); ++i) { atomicFD[i] = indexFieldData.load(indexReader.leaves().get(i)); subs[i] = atomicFD[i].getOrdinalsValues(); @@ -83,11 +83,11 @@ public static IndexOrdinalsFieldData buildEmpty(IndexSettings indexSettings, fin assert indexReader.leaves().size() > 1; final AtomicOrdinalsFieldData[] atomicFD = new AtomicOrdinalsFieldData[indexReader.leaves().size()]; - final RandomAccessOrds[] subs = new RandomAccessOrds[indexReader.leaves().size()]; + final SortedSetDocValues[] subs = new SortedSetDocValues[indexReader.leaves().size()]; for (int i = 0; i < indexReader.leaves().size(); ++i) { atomicFD[i] = new AbstractAtomicOrdinalsFieldData(AbstractAtomicOrdinalsFieldData.DEFAULT_SCRIPT_FUNCTION) { @Override - public RandomAccessOrds getOrdinalsValues() { + public SortedSetDocValues getOrdinalsValues() { return DocValues.emptySortedSet(); } diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/InternalGlobalOrdinalsIndexFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/InternalGlobalOrdinalsIndexFieldData.java index 284e22b21d4fa..23ecc06fed688 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/InternalGlobalOrdinalsIndexFieldData.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/InternalGlobalOrdinalsIndexFieldData.java @@ -20,7 +20,7 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.MultiDocValues.OrdinalMap; -import org.apache.lucene.index.RandomAccessOrds; +import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.util.Accountable; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.fielddata.AtomicOrdinalsFieldData; @@ -36,10 +36,10 @@ final class InternalGlobalOrdinalsIndexFieldData extends GlobalOrdinalsIndexFieldData { private final Atomic[] atomicReaders; - private final Function> scriptFunction; + private final Function> scriptFunction; InternalGlobalOrdinalsIndexFieldData(IndexSettings indexSettings, String fieldName, AtomicOrdinalsFieldData[] segmentAfd, - OrdinalMap ordinalMap, long memorySizeInBytes, Function> scriptFunction) { + OrdinalMap ordinalMap, long memorySizeInBytes, Function> scriptFunction) { super(indexSettings, fieldName, memorySizeInBytes); this.atomicReaders = new Atomic[segmentAfd.length]; for (int i = 0; i < segmentAfd.length; i++) { @@ -67,13 +67,13 @@ private Atomic(AtomicOrdinalsFieldData afd, OrdinalMap ordinalMap, int segmentIn } @Override - public RandomAccessOrds getOrdinalsValues() { - final RandomAccessOrds values = afd.getOrdinalsValues(); + public SortedSetDocValues getOrdinalsValues() { + final SortedSetDocValues values = afd.getOrdinalsValues(); if (values.getValueCount() == ordinalMap.getValueCount()) { // segment ordinals match global ordinals return values; } - final RandomAccessOrds[] bytesValues = new RandomAccessOrds[atomicReaders.length]; + final SortedSetDocValues[] bytesValues = new SortedSetDocValues[atomicReaders.length]; for (int i = 0; i < bytesValues.length; i++) { bytesValues[i] = atomicReaders[i].afd.getOrdinalsValues(); } diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractAtomicOrdinalsFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractAtomicOrdinalsFieldData.java index 7959bf3578a5d..9130e19fb26e1 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractAtomicOrdinalsFieldData.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractAtomicOrdinalsFieldData.java @@ -20,7 +20,7 @@ package org.elasticsearch.index.fielddata.plain; import org.apache.lucene.index.DocValues; -import org.apache.lucene.index.RandomAccessOrds; +import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.util.Accountable; import org.elasticsearch.index.fielddata.AtomicOrdinalsFieldData; import org.elasticsearch.index.fielddata.FieldData; @@ -34,13 +34,13 @@ public abstract class AbstractAtomicOrdinalsFieldData implements AtomicOrdinalsFieldData { - public static final Function> DEFAULT_SCRIPT_FUNCTION = - ((Function) FieldData::toString) + public static final Function> DEFAULT_SCRIPT_FUNCTION = + ((Function) FieldData::toString) .andThen(ScriptDocValues.Strings::new); - private final Function> scriptFunction; + private final Function> scriptFunction; - protected AbstractAtomicOrdinalsFieldData(Function> scriptFunction) { + protected AbstractAtomicOrdinalsFieldData(Function> scriptFunction) { this.scriptFunction = scriptFunction; } @@ -72,7 +72,7 @@ public void close() { } @Override - public RandomAccessOrds getOrdinalsValues() { + public SortedSetDocValues getOrdinalsValues() { return DocValues.emptySortedSet(); } }; diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/DocValuesIndexFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/DocValuesIndexFieldData.java index c77ceb57457ea..5378a201d44ef 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/DocValuesIndexFieldData.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/plain/DocValuesIndexFieldData.java @@ -22,6 +22,7 @@ import org.apache.logging.log4j.Logger; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.RandomAccessOrds; +import org.apache.lucene.index.SortedSetDocValues; import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexSettings; @@ -75,14 +76,14 @@ public static class Builder implements IndexFieldData.Builder { private static final Set BINARY_INDEX_FIELD_NAMES = unmodifiableSet(newHashSet(UidFieldMapper.NAME, IdFieldMapper.NAME)); private NumericType numericType; - private Function> scriptFunction = AbstractAtomicOrdinalsFieldData.DEFAULT_SCRIPT_FUNCTION; + private Function> scriptFunction = AbstractAtomicOrdinalsFieldData.DEFAULT_SCRIPT_FUNCTION; public Builder numericType(NumericType type) { this.numericType = type; return this; } - public Builder scriptFunction(Function> scriptFunction) { + public Builder scriptFunction(Function> scriptFunction) { this.scriptFunction = scriptFunction; return this; } diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/SortedSetDVBytesAtomicFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/SortedSetDVBytesAtomicFieldData.java index 114031cd3553b..d336c1d5cd7ce 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/SortedSetDVBytesAtomicFieldData.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/plain/SortedSetDVBytesAtomicFieldData.java @@ -21,10 +21,9 @@ import org.apache.lucene.index.DocValues; import org.apache.lucene.index.LeafReader; -import org.apache.lucene.index.RandomAccessOrds; +import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.util.Accountable; import org.elasticsearch.index.fielddata.AtomicFieldData; -import org.elasticsearch.index.fielddata.FieldData; import org.elasticsearch.index.fielddata.ScriptDocValues; import java.io.IOException; @@ -40,7 +39,7 @@ public final class SortedSetDVBytesAtomicFieldData extends AbstractAtomicOrdinal private final LeafReader reader; private final String field; - SortedSetDVBytesAtomicFieldData(LeafReader reader, String field, Function> scriptFunction) { super(scriptFunction); this.reader = reader; @@ -48,9 +47,9 @@ public final class SortedSetDVBytesAtomicFieldData extends AbstractAtomicOrdinal } @Override - public RandomAccessOrds getOrdinalsValues() { + public SortedSetDocValues getOrdinalsValues() { try { - return FieldData.maybeSlowRandomAccessOrds(DocValues.getSortedSet(reader, field)); + return DocValues.getSortedSet(reader, field); } catch (IOException e) { throw new IllegalStateException("cannot load docvalues", e); } diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/SortedSetDVOrdinalsIndexFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/SortedSetDVOrdinalsIndexFieldData.java index 2c59e8559c272..ea076d476de04 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/SortedSetDVOrdinalsIndexFieldData.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/plain/SortedSetDVOrdinalsIndexFieldData.java @@ -21,10 +21,10 @@ import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.RandomAccessOrds; +import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.search.SortField; -import org.apache.lucene.search.SortedSetSortField; import org.apache.lucene.search.SortedSetSelector; +import org.apache.lucene.search.SortedSetSortField; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.common.Nullable; import org.elasticsearch.index.IndexSettings; @@ -46,10 +46,10 @@ public class SortedSetDVOrdinalsIndexFieldData extends DocValuesIndexFieldData i private final IndexSettings indexSettings; private final IndexFieldDataCache cache; private final CircuitBreakerService breakerService; - private final Function> scriptFunction; + private final Function> scriptFunction; public SortedSetDVOrdinalsIndexFieldData(IndexSettings indexSettings, IndexFieldDataCache cache, String fieldName, - CircuitBreakerService breakerService, Function> scriptFunction) { + CircuitBreakerService breakerService, Function> scriptFunction) { super(indexSettings.getIndex(), fieldName); this.indexSettings = indexSettings; this.cache = cache; diff --git a/core/src/main/java/org/elasticsearch/index/mapper/CustomDocValuesField.java b/core/src/main/java/org/elasticsearch/index/mapper/CustomDocValuesField.java index 8e6cee222d4cd..60fbfc0698cda 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/CustomDocValuesField.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/CustomDocValuesField.java @@ -53,11 +53,6 @@ public IndexableFieldType fieldType() { return TYPE; } - @Override - public float boost() { - return 1f; - } - @Override public String stringValue() { return null; diff --git a/core/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java index 8b7355dca4b4a..39280bcee20d9 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java @@ -277,7 +277,7 @@ public ObjectMapper findNestedObjectMapper(int nestedDocId, SearchContext sc, Le } // We can pass down 'null' as acceptedDocs, because nestedDocId is a doc to be fetched and // therefor is guaranteed to be a live doc. - final Weight nestedWeight = filter.createWeight(sc.searcher(), false); + final Weight nestedWeight = filter.createWeight(sc.searcher(), false, 1f); Scorer scorer = nestedWeight.scorer(context); if (scorer == null) { continue; diff --git a/core/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java index 0ed093b3a884f..b8c958f41f30a 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java @@ -21,7 +21,6 @@ import com.carrotsearch.hppc.cursors.ObjectCursor; import com.carrotsearch.hppc.cursors.ObjectObjectCursor; -import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.IndexableField; @@ -286,12 +285,6 @@ public Mapper parse(ParseContext context) throws IOException { try { parseCreateField(context, fields); for (IndexableField field : fields) { - if (!customBoost() - // don't set boosts eg. on dv fields - && field.fieldType().indexOptions() != IndexOptions.NONE - && indexCreatedVersion.before(Version.V_5_0_0_alpha1)) { - ((Field)(field)).setBoost(fieldType().boost()); - } context.doc().add(field); } } catch (Exception e) { @@ -306,13 +299,6 @@ public Mapper parse(ParseContext context) throws IOException { */ protected abstract void parseCreateField(ParseContext context, List fields) throws IOException; - /** - * Derived classes can override it to specify that boost value is set by derived classes. - */ - protected boolean customBoost() { - return false; - } - @Override public Iterator iterator() { return multiFields.iterator(); diff --git a/core/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java index c46fe227d6fdc..2f09027c4ff40 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java @@ -31,7 +31,6 @@ import org.apache.lucene.spatial.prefix.tree.PackedQuadPrefixTree; import org.apache.lucene.spatial.prefix.tree.QuadPrefixTree; import org.apache.lucene.spatial.prefix.tree.SpatialPrefixTree; -import org.elasticsearch.Version; import org.elasticsearch.action.fieldstats.FieldStats; import org.elasticsearch.common.Explicit; import org.elasticsearch.common.geo.GeoUtils; @@ -463,10 +462,6 @@ public Mapper parse(ParseContext context) throws IOException { return null; } for (Field field : fields) { - if (!customBoost() && - fieldType.boost() != 1f && Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0_alpha1)) { - field.setBoost(fieldType().boost()); - } context.doc().add(field); } } catch (Exception e) { diff --git a/core/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java index a15432c635dea..5dfa0a9d04757 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java @@ -27,7 +27,7 @@ import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.PointValues; -import org.apache.lucene.index.RandomAccessOrds; +import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.Query; import org.apache.lucene.util.BytesRef; @@ -233,15 +233,31 @@ public FieldStats.Ip stats(IndexReader reader) throws IOException { public static final class IpScriptDocValues extends ScriptDocValues { - private final RandomAccessOrds values; + private final SortedSetDocValues values; + private int count; + private int lastIndex; + private int doc; - public IpScriptDocValues(RandomAccessOrds values) { + public IpScriptDocValues(SortedSetDocValues values) { this.values = values; } @Override public void setNextDocId(int docId) { - values.setDocument(docId); + count = 0; + try { + if (values.advanceExact(docId)) { + for (long ord = values.nextOrd(); ord != SortedSetDocValues.NO_MORE_ORDS; ord = values.nextOrd()) { + count++; + } + boolean hasValue = values.advanceExact(docId); + assert hasValue; + doc = docId; + lastIndex = -1; + } + } catch (IOException e) { + throw new RuntimeException(e); + } } public String getValue() { @@ -254,15 +270,27 @@ public String getValue() { @Override public String get(int index) { - BytesRef encoded = values.lookupOrd(values.ordAt(0)); - InetAddress address = InetAddressPoint.decode( - Arrays.copyOfRange(encoded.bytes, encoded.offset, encoded.offset + encoded.length)); - return InetAddresses.toAddrString(address); + try { + if (index <= lastIndex) { + boolean hasValue = values.advanceExact(doc); + assert hasValue; + lastIndex = -1; + } + for (int i = lastIndex + 1; i < index; ++i) { + values.nextOrd(); + } + BytesRef encoded = values.lookupOrd(values.nextOrd()); + InetAddress address = InetAddressPoint.decode( + Arrays.copyOfRange(encoded.bytes, encoded.offset, encoded.offset + encoded.length)); + return InetAddresses.toAddrString(address); + } catch (IOException e) { + throw new RuntimeException(e); + } } @Override public int size() { - return values.cardinality(); + return count; } } diff --git a/core/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java index 226ab905a2790..c69e68b24b6b3 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java @@ -219,7 +219,7 @@ public Query termQuery(Object value, QueryShardContext context) { } @Override - public Query termsQuery(List values, QueryShardContext context) { + public Query termsQuery(List values, QueryShardContext context) { failIfNotIndexed(); List scaledValues = new ArrayList<>(values.size()); for (Object value : values) { @@ -561,26 +561,30 @@ public SortedNumericDoubleValues getDoubleValues() { if (singleValues != null) { return FieldData.singleton(new NumericDoubleValues() { @Override - public double get(int docID) { - return singleValues.get(docID) * scalingFactorInverse; + public boolean advanceExact(int doc) throws IOException { + return singleValues.advanceExact(doc); } - }, DocValues.unwrapSingletonBits(values)); + @Override + public double doubleValue() throws IOException { + return singleValues.longValue() * scalingFactorInverse; + } + }); } else { return new SortedNumericDoubleValues() { @Override - public double valueAt(int index) { - return values.valueAt(index) * scalingFactorInverse; + public boolean advanceExact(int target) throws IOException { + return values.advanceExact(target); } @Override - public void setDocument(int doc) { - values.setDocument(doc); + public double nextValue() throws IOException { + return values.nextValue() * scalingFactorInverse; } @Override - public int count() { - return values.count(); + public int docValueCount() { + return values.docValueCount(); } }; } From 992fd944fec9b7bfc5cc49d62fce7420e780268c Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Wed, 5 Apr 2017 15:23:33 +0200 Subject: [PATCH 12/87] Deprecate the `base` similarity. --- .../index/similarity/SimilarityService.java | 16 +++++++++------- docs/reference/index-modules/similarity.asciidoc | 12 ++++-------- .../migration/migrate_6_0/settings.asciidoc | 8 +++++++- 3 files changed, 20 insertions(+), 16 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/index/similarity/SimilarityService.java b/core/src/main/java/org/elasticsearch/index/similarity/SimilarityService.java index e8203af852317..2f3abf7a61bc5 100644 --- a/core/src/main/java/org/elasticsearch/index/similarity/SimilarityService.java +++ b/core/src/main/java/org/elasticsearch/index/similarity/SimilarityService.java @@ -23,6 +23,8 @@ import org.apache.lucene.search.similarities.Similarity; import org.elasticsearch.Version; import org.elasticsearch.common.TriFunction; +import org.elasticsearch.common.logging.DeprecationLogger; +import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.AbstractIndexComponent; import org.elasticsearch.index.IndexModule; @@ -36,9 +38,9 @@ public final class SimilarityService extends AbstractIndexComponent { + private static final DeprecationLogger DEPRECATION_LOGGER = new DeprecationLogger(Loggers.getLogger(SimilarityService.class)); public static final String DEFAULT_SIMILARITY = "BM25"; private final Similarity defaultSimilarity; - private final Similarity baseSimilarity; private final Map similarities; private static final Map> DEFAULTS; public static final Map> BUILT_IN; @@ -95,14 +97,14 @@ public SimilarityService(IndexSettings indexSettings, this.similarities = providers; defaultSimilarity = (providers.get("default") != null) ? providers.get("default").get() : providers.get(SimilarityService.DEFAULT_SIMILARITY).get(); - // Expert users can configure the base type as being different to default, but out-of-box we use default. - baseSimilarity = (providers.get("base") != null) ? providers.get("base").get() : - defaultSimilarity; + if (providers.get("base") != null) { + DEPRECATION_LOGGER.deprecated("The [base] similarity is ignored since query normalization and coords have been removed"); + } } public Similarity similarity(MapperService mapperService) { // TODO we can maybe factor out MapperService here entirely by introducing an interface for the lookup? - return (mapperService != null) ? new PerFieldSimilarity(defaultSimilarity, baseSimilarity, mapperService) : + return (mapperService != null) ? new PerFieldSimilarity(defaultSimilarity, mapperService) : defaultSimilarity; } @@ -134,8 +136,8 @@ static class PerFieldSimilarity extends PerFieldSimilarityWrapper { private final Similarity defaultSimilarity; private final MapperService mapperService; - PerFieldSimilarity(Similarity defaultSimilarity, Similarity baseSimilarity, MapperService mapperService) { - super(baseSimilarity); + PerFieldSimilarity(Similarity defaultSimilarity, MapperService mapperService) { + super(); this.defaultSimilarity = defaultSimilarity; this.mapperService = mapperService; } diff --git a/docs/reference/index-modules/similarity.asciidoc b/docs/reference/index-modules/similarity.asciidoc index 7930ed573b462..61333c41a0f3d 100644 --- a/docs/reference/index-modules/similarity.asciidoc +++ b/docs/reference/index-modules/similarity.asciidoc @@ -165,14 +165,10 @@ Type name: `LMJelinekMercer` [float] [[default-base]] -==== Default and Base Similarities +==== Default Similarity By default, Elasticsearch will use whatever similarity is configured as -`default`. However, the similarity functions `queryNorm()` and `coord()` -are not per-field. Consequently, for expert users wanting to change the -implementation used for these two methods, while not changing the -`default`, it is possible to configure a similarity with the name -`base`. This similarity will then be used for the two methods. +`default`. You can change the default similarity for all fields in an index when it is <>: @@ -185,7 +181,7 @@ PUT /my_index "index": { "similarity": { "default": { - "type": "classic" + "type": "boolean" } } } @@ -205,7 +201,7 @@ PUT /my_index/_settings "index": { "similarity": { "default": { - "type": "classic" + "type": "boolean" } } } diff --git a/docs/reference/migration/migrate_6_0/settings.asciidoc b/docs/reference/migration/migrate_6_0/settings.asciidoc index ec25ffb601cad..ea11e23ebffc7 100644 --- a/docs/reference/migration/migrate_6_0/settings.asciidoc +++ b/docs/reference/migration/migrate_6_0/settings.asciidoc @@ -33,4 +33,10 @@ will use the best `store` implementation for your operating system. The blocking TCP client, blocking TCP server, and blocking HTTP server have been removed. As a consequence, the `network.tcp.blocking_server`, `network.tcp.blocking_client`, `network.tcp.blocking`,`transport.tcp.blocking_client`, `transport.tcp.blocking_server`, -and `http.tcp.blocking_server` settings are not recognized anymore. \ No newline at end of file +and `http.tcp.blocking_server` settings are not recognized anymore. + +==== Similarity settings + +The `base` similarity is now ignored as coords and query normalization have +been removed. If provided, this setting will be ignored and issue a +deprecation warning. From d6fe87297bbc0bf0ca06cda52ba20f71441081a7 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Wed, 5 Apr 2017 15:25:45 +0200 Subject: [PATCH 13/87] More fixes. --- .../index/query/ScriptQueryBuilder.java | 28 ++-- .../index/query/SimpleQueryParser.java | 6 +- .../index/search/MatchQuery.java | 2 +- .../index/search/MultiMatchQuery.java | 5 +- .../index/shard/FilterDocValuesProducer.java | 157 ------------------ .../index/shard/IndexSearcherWrapper.java | 7 +- 6 files changed, 25 insertions(+), 180 deletions(-) delete mode 100644 core/src/main/java/org/elasticsearch/index/shard/FilterDocValuesProducer.java diff --git a/core/src/main/java/org/elasticsearch/index/query/ScriptQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/ScriptQueryBuilder.java index f12e9b06767f5..4ad2e7824ebc9 100644 --- a/core/src/main/java/org/elasticsearch/index/query/ScriptQueryBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/query/ScriptQueryBuilder.java @@ -20,11 +20,14 @@ package org.elasticsearch.index.query; import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.search.ConstantScoreScorer; +import org.apache.lucene.search.ConstantScoreWeight; +import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; -import org.apache.lucene.search.RandomAccessWeight; +import org.apache.lucene.search.Scorer; +import org.apache.lucene.search.TwoPhaseIterator; import org.apache.lucene.search.Weight; -import org.apache.lucene.util.Bits; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.io.stream.StreamInput; @@ -172,16 +175,18 @@ public int hashCode() { } @Override - public Weight createWeight(IndexSearcher searcher, boolean needsScores) throws IOException { - return new RandomAccessWeight(this) { + public Weight createWeight(IndexSearcher searcher, boolean needsScores, float boost) throws IOException { + return new ConstantScoreWeight(this, boost) { + @Override - protected Bits getMatchingDocs(final LeafReaderContext context) throws IOException { + public Scorer scorer(LeafReaderContext context) throws IOException { + DocIdSetIterator approximation = DocIdSetIterator.all(context.reader().maxDoc()); final LeafSearchScript leafScript = searchScript.getLeafSearchScript(context); - return new Bits() { + TwoPhaseIterator twoPhase = new TwoPhaseIterator(approximation) { @Override - public boolean get(int doc) { - leafScript.setDocument(doc); + public boolean matches() throws IOException { + leafScript.setDocument(approximation.docID()); Object val = leafScript.run(); if (val == null) { return false; @@ -196,11 +201,12 @@ public boolean get(int doc) { } @Override - public int length() { - return context.reader().maxDoc(); + public float matchCost() { + // TODO: how can we compute this? + return 1000f; } - }; + return new ConstantScoreScorer(this, score(), twoPhase); } }; } diff --git a/core/src/main/java/org/elasticsearch/index/query/SimpleQueryParser.java b/core/src/main/java/org/elasticsearch/index/query/SimpleQueryParser.java index ddc1c620e3d58..6aa2bd843e2dc 100644 --- a/core/src/main/java/org/elasticsearch/index/query/SimpleQueryParser.java +++ b/core/src/main/java/org/elasticsearch/index/query/SimpleQueryParser.java @@ -78,7 +78,6 @@ protected Query newTermQuery(Term term) { @Override public Query newDefaultQuery(String text) { BooleanQuery.Builder bq = new BooleanQuery.Builder(); - bq.setDisableCoord(true); for (Map.Entry entry : weights.entrySet()) { try { Query q = createBooleanQuery(entry.getKey(), text, super.getDefaultOperator()); @@ -99,7 +98,6 @@ public Query newDefaultQuery(String text) { @Override public Query newFuzzyQuery(String text, int fuzziness) { BooleanQuery.Builder bq = new BooleanQuery.Builder(); - bq.setDisableCoord(true); for (Map.Entry entry : weights.entrySet()) { final String fieldName = entry.getKey(); try { @@ -116,7 +114,6 @@ public Query newFuzzyQuery(String text, int fuzziness) { @Override public Query newPhraseQuery(String text, int slop) { BooleanQuery.Builder bq = new BooleanQuery.Builder(); - bq.setDisableCoord(true); for (Map.Entry entry : weights.entrySet()) { try { String field = entry.getKey(); @@ -146,7 +143,6 @@ public Query newPhraseQuery(String text, int slop) { @Override public Query newPrefixQuery(String text) { BooleanQuery.Builder bq = new BooleanQuery.Builder(); - bq.setDisableCoord(true); for (Map.Entry entry : weights.entrySet()) { final String fieldName = entry.getKey(); try { @@ -244,7 +240,7 @@ private Query newPossiblyAnalyzedQuery(String field, String termStr) { innerBuilder.add(new BooleanClause(new PrefixQuery(new Term(field, token)), BooleanClause.Occur.SHOULD)); } - posQuery = innerBuilder.setDisableCoord(true).build(); + posQuery = innerBuilder.build(); } builder.add(new BooleanClause(posQuery, getDefaultOperator())); } diff --git a/core/src/main/java/org/elasticsearch/index/search/MatchQuery.java b/core/src/main/java/org/elasticsearch/index/search/MatchQuery.java index 21d084d5d2f22..bc600b0a0b115 100644 --- a/core/src/main/java/org/elasticsearch/index/search/MatchQuery.java +++ b/core/src/main/java/org/elasticsearch/index/search/MatchQuery.java @@ -405,7 +405,7 @@ public Query createCommonTermsQuery(String field, String queryText, Occur highFr private Query boolToExtendedCommonTermsQuery(BooleanQuery bq, Occur highFreqOccur, Occur lowFreqOccur, float maxTermFrequency, MappedFieldType fieldType) { ExtendedCommonTermsQuery query = new ExtendedCommonTermsQuery(highFreqOccur, lowFreqOccur, maxTermFrequency, - bq.isCoordDisabled(), fieldType); + fieldType); for (BooleanClause clause : bq.clauses()) { if (!(clause.getQuery() instanceof TermQuery)) { return bq; diff --git a/core/src/main/java/org/elasticsearch/index/search/MultiMatchQuery.java b/core/src/main/java/org/elasticsearch/index/search/MultiMatchQuery.java index 15e8561c8fd65..7c1f91d3587d5 100644 --- a/core/src/main/java/org/elasticsearch/index/search/MultiMatchQuery.java +++ b/core/src/main/java/org/elasticsearch/index/search/MultiMatchQuery.java @@ -305,9 +305,9 @@ static Query blendTerms(QueryShardContext context, BytesRef[] values, Float comm terms = Arrays.copyOf(terms, i); blendedBoost = Arrays.copyOf(blendedBoost, i); if (commonTermsCutoff != null) { - queries.add(BlendedTermQuery.commonTermsBlendedQuery(terms, blendedBoost, false, commonTermsCutoff)); + queries.add(BlendedTermQuery.commonTermsBlendedQuery(terms, blendedBoost, commonTermsCutoff)); } else if (tieBreaker == 1.0f) { - queries.add(BlendedTermQuery.booleanBlendedQuery(terms, blendedBoost, false)); + queries.add(BlendedTermQuery.booleanBlendedQuery(terms, blendedBoost)); } else { queries.add(BlendedTermQuery.dismaxBlendedQuery(terms, blendedBoost, tieBreaker)); } @@ -319,7 +319,6 @@ static Query blendTerms(QueryShardContext context, BytesRef[] values, Float comm // however their score contribution will be different // TODO: can we improve this? BooleanQuery.Builder bq = new BooleanQuery.Builder(); - bq.setDisableCoord(true); for (Query query : queries) { bq.add(query, Occur.SHOULD); } diff --git a/core/src/main/java/org/elasticsearch/index/shard/FilterDocValuesProducer.java b/core/src/main/java/org/elasticsearch/index/shard/FilterDocValuesProducer.java deleted file mode 100644 index 149915458ee82..0000000000000 --- a/core/src/main/java/org/elasticsearch/index/shard/FilterDocValuesProducer.java +++ /dev/null @@ -1,157 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.shard; - -import org.apache.lucene.codecs.DocValuesProducer; -import org.apache.lucene.index.BinaryDocValues; -import org.apache.lucene.index.FieldInfo; -import org.apache.lucene.index.NumericDocValues; -import org.apache.lucene.index.SortedDocValues; -import org.apache.lucene.index.SortedNumericDocValues; -import org.apache.lucene.index.SortedSetDocValues; -import org.apache.lucene.util.Accountable; -import org.apache.lucene.util.Bits; - -import java.io.IOException; -import java.util.Collection; -import java.util.Collections; - -// TODO: move this to lucene's FilterCodecReader - -/** - * Base class for filtering DocValuesProducer implementations. - *

- * NOTE: just like with DocValuesProducer, the default {@link #getMergeInstance()} - * is unoptimized. overriding this method when possible can improve performance. - */ -class FilterDocValuesProducer extends DocValuesProducer { - /** The underlying Producer instance. */ - protected final DocValuesProducer in; - - /** - * Creates a new FilterDocValuesProducer - * @param in the underlying producer. - */ - FilterDocValuesProducer(DocValuesProducer in) { - this.in = in; - } - - @Override - public void close() throws IOException { - in.close(); - } - - @Override - public long ramBytesUsed() { - return in.ramBytesUsed(); - } - - @Override - public Collection getChildResources() { - return in.getChildResources(); - } - - @Override - public NumericDocValues getNumeric(FieldInfo field) throws IOException { - return in.getNumeric(field); - } - - @Override - public BinaryDocValues getBinary(FieldInfo field) throws IOException { - return in.getBinary(field); - } - - @Override - public SortedDocValues getSorted(FieldInfo field) throws IOException { - return in.getSorted(field); - } - - @Override - public SortedNumericDocValues getSortedNumeric(FieldInfo field) throws IOException { - return in.getSortedNumeric(field); - } - - @Override - public SortedSetDocValues getSortedSet(FieldInfo field) throws IOException { - return in.getSortedSet(field); - } - - @Override - public Bits getDocsWithField(FieldInfo field) throws IOException { - return in.getDocsWithField(field); - } - - @Override - public void checkIntegrity() throws IOException { - in.checkIntegrity(); - } - - // TODO: move this out somewhere else (and can fix all these null producers in lucene?) - // we shouldn't need nullness for any reason. - - public static final DocValuesProducer EMPTY = new DocValuesProducer() { - - @Override - public void close() throws IOException {} - - @Override - public long ramBytesUsed() { - return 0; - } - - @Override - public Collection getChildResources() { - return Collections.emptyList(); - } - - @Override - public NumericDocValues getNumeric(FieldInfo field) throws IOException { - throw new IllegalStateException(); // we don't have any docvalues - } - - @Override - public BinaryDocValues getBinary(FieldInfo field) throws IOException { - throw new IllegalStateException(); // we don't have any docvalues - } - - @Override - public SortedDocValues getSorted(FieldInfo field) throws IOException { - throw new IllegalStateException(); // we don't have any docvalues - } - - @Override - public SortedNumericDocValues getSortedNumeric(FieldInfo field) throws IOException { - throw new IllegalStateException(); // we don't have any docvalues - } - - @Override - public SortedSetDocValues getSortedSet(FieldInfo field) throws IOException { - throw new IllegalStateException(); // we don't have any docvalues - } - - @Override - public Bits getDocsWithField(FieldInfo field) throws IOException { - throw new IllegalStateException(); // we don't have any docvalues - } - - @Override - public void checkIntegrity() throws IOException {} - }; -} diff --git a/core/src/main/java/org/elasticsearch/index/shard/IndexSearcherWrapper.java b/core/src/main/java/org/elasticsearch/index/shard/IndexSearcherWrapper.java index 5603001a293e2..5f6065ca66978 100644 --- a/core/src/main/java/org/elasticsearch/index/shard/IndexSearcherWrapper.java +++ b/core/src/main/java/org/elasticsearch/index/shard/IndexSearcherWrapper.java @@ -74,7 +74,7 @@ public final Engine.Searcher wrap(Engine.Searcher engineSearcher) throws IOExcep NonClosingReaderWrapper nonClosingReaderWrapper = new NonClosingReaderWrapper(engineSearcher.getDirectoryReader()); DirectoryReader reader = wrap(nonClosingReaderWrapper); if (reader != nonClosingReaderWrapper) { - if (reader.getCoreCacheKey() != elasticsearchDirectoryReader.getCoreCacheKey()) { + if (reader.getReaderCacheHelper() != elasticsearchDirectoryReader.getReaderCacheHelper()) { throw new IllegalStateException("wrapped directory reader doesn't delegate IndexReader#getCoreCacheKey, wrappers must override this method and delegate" + " to the original readers core cache key. Wrapped readers can't be used as cache keys since their are used only per request which would lead to subtle bugs"); } @@ -136,9 +136,10 @@ protected void doClose() throws IOException { } @Override - public Object getCoreCacheKey() { - return in.getCoreCacheKey(); + public CacheHelper getReaderCacheHelper() { + return in.getReaderCacheHelper(); } + } } From 131347d34928b95d0326eaaaf4f3e68147878551 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Wed, 5 Apr 2017 15:34:12 +0200 Subject: [PATCH 14/87] Fix errors in the expressions module. --- .../expression/CountMethodValueSource.java | 13 ++++++++----- .../script/expression/DateMethodValueSource.java | 16 ++++++++++------ .../script/expression/DateObjectValueSource.java | 16 ++++++++++------ .../expression/EmptyMemberValueSource.java | 5 ++--- .../script/expression/FieldDataValueSource.java | 8 ++++++-- .../script/expression/GeoEmptyValueSource.java | 5 ++--- .../expression/GeoLatitudeValueSource.java | 9 ++++----- .../expression/GeoLongitudeValueSource.java | 9 ++++----- 8 files changed, 46 insertions(+), 35 deletions(-) diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/CountMethodValueSource.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/CountMethodValueSource.java index 1aa85ef685b2b..6a36ff7a53943 100644 --- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/CountMethodValueSource.java +++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/CountMethodValueSource.java @@ -50,11 +50,14 @@ public FunctionValues getValues(Map context, LeafReaderContext leaf) throws IOEx final SortedNumericDoubleValues values = leafData.getDoubleValues(); return new DoubleDocValues(this) { - @Override - public double doubleVal(int doc) { - values.setDocument(doc); - return values.count(); - } + @Override + public double doubleVal(int doc) throws IOException { + if (values.advanceExact(doc)) { + return values.docValueCount(); + } else { + return 0; + } + } }; } diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/DateMethodValueSource.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/DateMethodValueSource.java index 98909f4401a68..b0bc7c203b63b 100644 --- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/DateMethodValueSource.java +++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/DateMethodValueSource.java @@ -56,12 +56,16 @@ public FunctionValues getValues(Map context, LeafReaderContext leaf) throws IOEx final Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC"), Locale.ROOT); NumericDoubleValues docValues = multiValueMode.select(leafData.getDoubleValues(), 0d); return new DoubleDocValues(this) { - @Override - public double doubleVal(int docId) { - long millis = (long)docValues.get(docId); - calendar.setTimeInMillis(millis); - return calendar.get(calendarType); - } + @Override + public double doubleVal(int docId) throws IOException { + if (docValues.advanceExact(docId)) { + long millis = (long)docValues.doubleValue(); + calendar.setTimeInMillis(millis); + return calendar.get(calendarType); + } else { + return 0; + } + } }; } diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/DateObjectValueSource.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/DateObjectValueSource.java index a9f2018d39829..86abcbcbefabd 100644 --- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/DateObjectValueSource.java +++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/DateObjectValueSource.java @@ -58,12 +58,16 @@ public FunctionValues getValues(Map context, LeafReaderContext leaf) throws IOEx MutableDateTime joda = new MutableDateTime(0, DateTimeZone.UTC); NumericDoubleValues docValues = multiValueMode.select(leafData.getDoubleValues(), 0d); return new DoubleDocValues(this) { - @Override - public double doubleVal(int docId) { - long millis = (long)docValues.get(docId); - joda.setMillis(millis); - return function.applyAsInt(joda); - } + @Override + public double doubleVal(int docId) throws IOException { + if (docValues.advanceExact(docId)) { + long millis = (long)docValues.doubleValue(); + joda.setMillis(millis); + return function.applyAsInt(joda); + } else { + return 0; + } + } }; } diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/EmptyMemberValueSource.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/EmptyMemberValueSource.java index b4c8582e0d664..0c912c6b0d83c 100644 --- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/EmptyMemberValueSource.java +++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/EmptyMemberValueSource.java @@ -50,9 +50,8 @@ public FunctionValues getValues(Map context, LeafReaderContext leaf) throws IOEx final SortedNumericDoubleValues values = leafData.getDoubleValues(); return new DoubleDocValues(this) { @Override - public double doubleVal(int doc) { - values.setDocument(doc); - if (values.count() == 0) { + public double doubleVal(int doc) throws IOException { + if (values.advanceExact(doc)) { return 1; } else { return 0; diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/FieldDataValueSource.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/FieldDataValueSource.java index 3ac885e49dccc..f2aaced37658e 100644 --- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/FieldDataValueSource.java +++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/FieldDataValueSource.java @@ -71,8 +71,12 @@ public FunctionValues getValues(Map context, LeafReaderContext leaf) throws IOEx NumericDoubleValues docValues = multiValueMode.select(leafData.getDoubleValues(), 0d); return new DoubleDocValues(this) { @Override - public double doubleVal(int doc) { - return docValues.get(doc); + public double doubleVal(int doc) throws IOException { + if (docValues.advanceExact(doc)) { + return docValues.doubleValue(); + } else { + return 0; + } } }; } diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoEmptyValueSource.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoEmptyValueSource.java index 2a9b09ba10d3b..0b16aaf9dcde0 100644 --- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoEmptyValueSource.java +++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoEmptyValueSource.java @@ -48,9 +48,8 @@ public FunctionValues getValues(Map context, LeafReaderContext leaf) throws IOEx final MultiGeoPointValues values = leafData.getGeoPointValues(); return new DoubleDocValues(this) { @Override - public double doubleVal(int doc) { - values.setDocument(doc); - if (values.count() == 0) { + public double doubleVal(int doc) throws IOException { + if (values.advanceExact(doc)) { return 1; } else { return 0; diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoLatitudeValueSource.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoLatitudeValueSource.java index d23eceda2feb3..fd812dac5a3a8 100644 --- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoLatitudeValueSource.java +++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoLatitudeValueSource.java @@ -48,12 +48,11 @@ public FunctionValues getValues(Map context, LeafReaderContext leaf) throws IOEx final MultiGeoPointValues values = leafData.getGeoPointValues(); return new DoubleDocValues(this) { @Override - public double doubleVal(int doc) { - values.setDocument(doc); - if (values.count() == 0) { - return 0.0; + public double doubleVal(int doc) throws IOException { + if (values.advanceExact(doc)) { + return values.nextValue().getLat(); } else { - return values.valueAt(0).getLat(); + return 0.0; } } }; diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoLongitudeValueSource.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoLongitudeValueSource.java index 4fbc8fd936cef..fd05d92d62350 100644 --- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoLongitudeValueSource.java +++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoLongitudeValueSource.java @@ -48,12 +48,11 @@ public FunctionValues getValues(Map context, LeafReaderContext leaf) throws IOEx final MultiGeoPointValues values = leafData.getGeoPointValues(); return new DoubleDocValues(this) { @Override - public double doubleVal(int doc) { - values.setDocument(doc); - if (values.count() == 0) { - return 0.0; + public double doubleVal(int doc) throws IOException { + if (values.advanceExact(doc)) { + return values.nextValue().getLon(); } else { - return values.valueAt(0).getLon(); + return 0.0; } } }; From afdc68c749e1c9fe84cd691a2b542cdd761f1eb0 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Wed, 5 Apr 2017 15:39:10 +0200 Subject: [PATCH 15/87] Fix compile errors in the percolator module. --- .../percolator/PercolateQueryBuilder.java | 6 ++---- .../percolator/CandidateQueryTests.java | 12 ++++++++---- .../elasticsearch/percolator/QueryAnalyzerTests.java | 2 +- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java index d548a3f354ac5..9ebb65c32753c 100644 --- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java +++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java @@ -33,7 +33,6 @@ import org.apache.lucene.search.Query; import org.apache.lucene.search.Weight; import org.apache.lucene.store.RAMDirectory; -import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ResourceNotFoundException; @@ -485,10 +484,9 @@ private static PercolateQuery.QueryStore createStore(PercolatorFieldMapper.Field return docId -> null; } - Bits bits = leafReader.getDocsWithField(fieldType.queryBuilderField.name()); return docId -> { - if (bits.get(docId)) { - BytesRef qbSource = binaryDocValues.get(docId); + if (binaryDocValues.advanceExact(docId)) { + BytesRef qbSource = binaryDocValues.binaryValue(); if (qbSource.length > 0) { XContent xContent = PercolatorFieldMapper.QUERY_BUILDER_CONTENT_TYPE.xContent(); try (XContentParser sourceParser = xContent.createParser(context.getXContentRegistry(), qbSource.bytes, diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/CandidateQueryTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/CandidateQueryTests.java index 65005e957ac5c..8c594a0545f00 100644 --- a/modules/percolator/src/test/java/org/elasticsearch/percolator/CandidateQueryTests.java +++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/CandidateQueryTests.java @@ -79,6 +79,7 @@ import java.util.Collection; import java.util.Collections; import java.util.List; +import java.util.Set; import java.util.function.Function; import static org.hamcrest.Matchers.equalTo; @@ -245,7 +246,7 @@ public void testDuelSpecificQueries() throws Exception { addQuery(commonTermsQuery, documents); BlendedTermQuery blendedTermQuery = BlendedTermQuery.booleanBlendedQuery(new Term[]{new Term("field", "quick"), - new Term("field", "brown"), new Term("field", "fox")}, false); + new Term("field", "brown"), new Term("field", "fox")}); addQuery(blendedTermQuery, documents); SpanNearQuery spanNearQuery = new SpanNearQuery.Builder("field", true) @@ -360,11 +361,14 @@ private ControlQuery(MemoryIndex memoryIndex, PercolateQuery.QueryStore querySto } @Override - public Weight createWeight(IndexSearcher searcher, boolean needsScores) { - return new ConstantScoreWeight(this) { + public Weight createWeight(IndexSearcher searcher, boolean needsScores, float boost) { + return new Weight(this) { float _score; + @Override + public void extractTerms(Set terms) {} + @Override public Explanation explain(LeafReaderContext context, int doc) throws IOException { Scorer scorer = scorer(context); @@ -406,7 +410,7 @@ protected boolean match(int doc) { } } }; - return new FilterScorer(new ConstantScoreScorer(this, score(), memoryIndexIterator)) { + return new FilterScorer(new ConstantScoreScorer(this, 1f, memoryIndexIterator)) { @Override public float score() throws IOException { diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/QueryAnalyzerTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/QueryAnalyzerTests.java index 558d237f1259c..ea18bd38d12f1 100644 --- a/modules/percolator/src/test/java/org/elasticsearch/percolator/QueryAnalyzerTests.java +++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/QueryAnalyzerTests.java @@ -276,7 +276,7 @@ public void testExtractQueryMetadata_commonTermsQuery() { public void testExtractQueryMetadata_blendedTermQuery() { Term[] termsArr = new Term[]{new Term("_field", "_term1"), new Term("_field", "_term2")}; - BlendedTermQuery commonTermsQuery = BlendedTermQuery.booleanBlendedQuery(termsArr, false); + BlendedTermQuery commonTermsQuery = BlendedTermQuery.booleanBlendedQuery(termsArr); Result result = analyze(commonTermsQuery); assertThat(result.verified, is(true)); List terms = new ArrayList<>(result.terms); From c4ddd34f9524cb02e2d9267f529e7c77fb0b04e6 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Wed, 5 Apr 2017 15:44:34 +0200 Subject: [PATCH 16/87] Fix compile errors in the test-framework. --- .../main/java/org/elasticsearch/test/ESTestCase.java | 4 ---- .../org/elasticsearch/test/FieldMaskingReader.java | 4 ++-- .../elasticsearch/test/engine/MockEngineSupport.java | 5 ----- .../test/engine/ThrowingLeafReaderWrapper.java | 11 +++++++++++ 4 files changed, 13 insertions(+), 11 deletions(-) diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java index 587d390f300ca..3e2133e12b657 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java @@ -37,7 +37,6 @@ import org.apache.logging.log4j.status.StatusConsoleListener; import org.apache.logging.log4j.status.StatusData; import org.apache.logging.log4j.status.StatusLogger; -import org.apache.lucene.uninverting.UninvertingReader; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.LuceneTestCase.SuppressCodecs; import org.apache.lucene.util.TestRuleMarkFailure; @@ -387,9 +386,6 @@ public void log(StatusData data) { protected static void checkStaticState() throws Exception { MockPageCacheRecycler.ensureAllPagesAreReleased(); MockBigArrays.ensureAllArraysAreReleased(); - // field cache should NEVER get loaded. - String[] entries = UninvertingReader.getUninvertedStats(); - assertEquals("fieldcache must never be used, got=" + Arrays.toString(entries), 0, entries.length); // ensure no one changed the status logger level on us assertThat(StatusLogger.getLogger().getLevel(), equalTo(Level.WARN)); diff --git a/test/framework/src/main/java/org/elasticsearch/test/FieldMaskingReader.java b/test/framework/src/main/java/org/elasticsearch/test/FieldMaskingReader.java index 5ce620166c158..4a7c95cfd44f8 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/FieldMaskingReader.java +++ b/test/framework/src/main/java/org/elasticsearch/test/FieldMaskingReader.java @@ -45,7 +45,7 @@ protected DirectoryReader doWrapDirectoryReader(DirectoryReader in) throws IOExc } @Override - public Object getCoreCacheKey() { - return in.getCoreCacheKey(); + public CacheHelper getReaderCacheHelper() { + return in.getReaderCacheHelper(); } } \ No newline at end of file diff --git a/test/framework/src/main/java/org/elasticsearch/test/engine/MockEngineSupport.java b/test/framework/src/main/java/org/elasticsearch/test/engine/MockEngineSupport.java index fbc4352b1e28c..ab9d04f1556a0 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/engine/MockEngineSupport.java +++ b/test/framework/src/main/java/org/elasticsearch/test/engine/MockEngineSupport.java @@ -180,11 +180,6 @@ public DirectoryReaderWrapper(DirectoryReader in, SubReaderWrapper subReaderWrap this.subReaderWrapper = subReaderWrapper; } - @Override - public Object getCoreCacheKey() { - return in.getCoreCacheKey(); - } - } public Engine.Searcher wrapSearcher(String source, Engine.Searcher engineSearcher, IndexSearcher searcher, SearcherManager manager) { diff --git a/test/framework/src/main/java/org/elasticsearch/test/engine/ThrowingLeafReaderWrapper.java b/test/framework/src/main/java/org/elasticsearch/test/engine/ThrowingLeafReaderWrapper.java index 9da5d7515c76d..314f1b5285204 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/engine/ThrowingLeafReaderWrapper.java +++ b/test/framework/src/main/java/org/elasticsearch/test/engine/ThrowingLeafReaderWrapper.java @@ -195,4 +195,15 @@ public NumericDocValues getNormValues(String field) throws IOException { thrower.maybeThrow(Flags.Norms); return super.getNormValues(field); } + + + @Override + public CacheHelper getCoreCacheHelper() { + return in.getCoreCacheHelper(); + } + + @Override + public CacheHelper getReaderCacheHelper() { + return in.getReaderCacheHelper(); + } } From 0d01bca04afe429d29e345185f578113f8e82468 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Wed, 5 Apr 2017 16:21:18 +0200 Subject: [PATCH 17/87] More compile fixes. --- .../lucene/index/ElasticsearchLeafReader.java | 9 ++- .../common/lucene/search/Queries.java | 2 - .../function/FieldValueFactorFunction.java | 12 ++-- .../function/FiltersFunctionScoreQuery.java | 2 +- .../search/function/LeafScoreFunction.java | 2 +- .../search/function/RandomScoreFunction.java | 11 ++-- .../search/function/WeightFactorFunction.java | 2 +- .../uid/PerThreadIDAndVersionLookup.java | 16 +++--- .../functionscore/DecayFunctionBuilder.java | 57 ++++++++++--------- 9 files changed, 62 insertions(+), 51 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/common/lucene/index/ElasticsearchLeafReader.java b/core/src/main/java/org/elasticsearch/common/lucene/index/ElasticsearchLeafReader.java index 1094a7b1840cd..0f13ee43d30e7 100644 --- a/core/src/main/java/org/elasticsearch/common/lucene/index/ElasticsearchLeafReader.java +++ b/core/src/main/java/org/elasticsearch/common/lucene/index/ElasticsearchLeafReader.java @@ -49,8 +49,13 @@ public ShardId shardId() { } @Override - public Object getCoreCacheKey() { - return in.getCoreCacheKey(); + public CacheHelper getCoreCacheHelper() { + return in.getCoreCacheHelper(); + } + + @Override + public CacheHelper getReaderCacheHelper() { + return in.getReaderCacheHelper(); } public static ElasticsearchLeafReader getElasticsearchLeafReader(LeafReader reader) { diff --git a/core/src/main/java/org/elasticsearch/common/lucene/search/Queries.java b/core/src/main/java/org/elasticsearch/common/lucene/search/Queries.java index 68a02ed256d8f..a7a94cf206442 100644 --- a/core/src/main/java/org/elasticsearch/common/lucene/search/Queries.java +++ b/core/src/main/java/org/elasticsearch/common/lucene/search/Queries.java @@ -121,7 +121,6 @@ public static Query fixNegativeQueryIfNeeded(Query q) { if (isNegativeQuery(q)) { BooleanQuery bq = (BooleanQuery) q; BooleanQuery.Builder builder = new BooleanQuery.Builder(); - builder.setDisableCoord(bq.isCoordDisabled()); for (BooleanClause clause : bq) { builder.add(clause); } @@ -154,7 +153,6 @@ public static Query applyMinimumShouldMatch(BooleanQuery query, @Nullable String int msm = calculateMinShouldMatch(optionalClauses, minimumShouldMatch); if (0 < msm) { BooleanQuery.Builder builder = new BooleanQuery.Builder(); - builder.setDisableCoord(query.isCoordDisabled()); for (BooleanClause clause : query) { builder.add(clause); } diff --git a/core/src/main/java/org/elasticsearch/common/lucene/search/function/FieldValueFactorFunction.java b/core/src/main/java/org/elasticsearch/common/lucene/search/function/FieldValueFactorFunction.java index 3bc5542c2aa13..2c5fd5924597c 100644 --- a/core/src/main/java/org/elasticsearch/common/lucene/search/function/FieldValueFactorFunction.java +++ b/core/src/main/java/org/elasticsearch/common/lucene/search/function/FieldValueFactorFunction.java @@ -62,7 +62,7 @@ public FieldValueFactorFunction(String field, float boostFactor, Modifier modifi public LeafScoreFunction getLeafScoreFunction(LeafReaderContext ctx) { final SortedNumericDoubleValues values; if(indexFieldData == null) { - values = FieldData.emptySortedNumericDoubles(ctx.reader().maxDoc()); + values = FieldData.emptySortedNumericDoubles(); } else { values = this.indexFieldData.load(ctx).getDoubleValues(); } @@ -70,12 +70,10 @@ public LeafScoreFunction getLeafScoreFunction(LeafReaderContext ctx) { return new LeafScoreFunction() { @Override - public double score(int docId, float subQueryScore) { - values.setDocument(docId); - final int numValues = values.count(); + public double score(int docId, float subQueryScore) throws IOException { double value; - if (numValues > 0) { - value = values.valueAt(0); + if (values.advanceExact(docId)) { + value = values.nextValue(); } else if (missing != null) { value = missing; } else { @@ -91,7 +89,7 @@ public double score(int docId, float subQueryScore) { } @Override - public Explanation explainScore(int docId, Explanation subQueryScore) { + public Explanation explainScore(int docId, Explanation subQueryScore) throws IOException { String modifierStr = modifier != null ? modifier.toString() : ""; String defaultStr = missing != null ? "?:" + missing : ""; double score = score(docId, subQueryScore.getValue()); diff --git a/core/src/main/java/org/elasticsearch/common/lucene/search/function/FiltersFunctionScoreQuery.java b/core/src/main/java/org/elasticsearch/common/lucene/search/function/FiltersFunctionScoreQuery.java index 801f399a669d0..abf145406c518 100644 --- a/core/src/main/java/org/elasticsearch/common/lucene/search/function/FiltersFunctionScoreQuery.java +++ b/core/src/main/java/org/elasticsearch/common/lucene/search/function/FiltersFunctionScoreQuery.java @@ -271,7 +271,7 @@ public float score() throws IOException { return scoreCombiner.combine(subQueryScore, factor, maxBoost); } - protected double computeScore(int docId, float subQueryScore) { + protected double computeScore(int docId, float subQueryScore) throws IOException { double factor = 1d; switch(scoreMode) { case FIRST: diff --git a/core/src/main/java/org/elasticsearch/common/lucene/search/function/LeafScoreFunction.java b/core/src/main/java/org/elasticsearch/common/lucene/search/function/LeafScoreFunction.java index 7df35fa580fee..c00431a34bad0 100644 --- a/core/src/main/java/org/elasticsearch/common/lucene/search/function/LeafScoreFunction.java +++ b/core/src/main/java/org/elasticsearch/common/lucene/search/function/LeafScoreFunction.java @@ -26,7 +26,7 @@ /** Per-leaf {@link ScoreFunction}. */ public abstract class LeafScoreFunction { - public abstract double score(int docId, float subQueryScore); + public abstract double score(int docId, float subQueryScore) throws IOException; public abstract Explanation explainScore(int docId, Explanation subQueryScore) throws IOException; diff --git a/core/src/main/java/org/elasticsearch/common/lucene/search/function/RandomScoreFunction.java b/core/src/main/java/org/elasticsearch/common/lucene/search/function/RandomScoreFunction.java index 3810c16bc0ed4..43e71f4dff34c 100644 --- a/core/src/main/java/org/elasticsearch/common/lucene/search/function/RandomScoreFunction.java +++ b/core/src/main/java/org/elasticsearch/common/lucene/search/function/RandomScoreFunction.java @@ -25,6 +25,7 @@ import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.SortedBinaryDocValues; +import java.io.IOException; import java.util.Objects; /** @@ -68,14 +69,16 @@ public LeafScoreFunction getLeafScoreFunction(LeafReaderContext ctx) { return new LeafScoreFunction() { @Override - public double score(int docId, float subQueryScore) { - uidByteData.setDocument(docId); - int hash = StringHelper.murmurhash3_x86_32(uidByteData.valueAt(0), saltedSeed); + public double score(int docId, float subQueryScore) throws IOException { + if (uidByteData.advanceExact(docId) == false) { + throw new AssertionError("Document without a _uid"); + } + int hash = StringHelper.murmurhash3_x86_32(uidByteData.nextValue(), saltedSeed); return (hash & 0x00FFFFFF) / (float)(1 << 24); // only use the lower 24 bits to construct a float from 0.0-1.0 } @Override - public Explanation explainScore(int docId, Explanation subQueryScore) { + public Explanation explainScore(int docId, Explanation subQueryScore) throws IOException { return Explanation.match( CombineFunction.toFloat(score(docId, subQueryScore.getValue())), "random score function (seed: " + originalSeed + ")"); diff --git a/core/src/main/java/org/elasticsearch/common/lucene/search/function/WeightFactorFunction.java b/core/src/main/java/org/elasticsearch/common/lucene/search/function/WeightFactorFunction.java index 44c6245a812b1..c69a5efdd53a4 100644 --- a/core/src/main/java/org/elasticsearch/common/lucene/search/function/WeightFactorFunction.java +++ b/core/src/main/java/org/elasticsearch/common/lucene/search/function/WeightFactorFunction.java @@ -52,7 +52,7 @@ public LeafScoreFunction getLeafScoreFunction(LeafReaderContext ctx) throws IOEx final LeafScoreFunction leafFunction = scoreFunction.getLeafScoreFunction(ctx); return new LeafScoreFunction() { @Override - public double score(int docId, float subQueryScore) { + public double score(int docId, float subQueryScore) throws IOException { return leafFunction.score(docId, subQueryScore) * getWeight(); } diff --git a/core/src/main/java/org/elasticsearch/common/lucene/uid/PerThreadIDAndVersionLookup.java b/core/src/main/java/org/elasticsearch/common/lucene/uid/PerThreadIDAndVersionLookup.java index 6e4e3e1923d5c..14090d3633d40 100644 --- a/core/src/main/java/org/elasticsearch/common/lucene/uid/PerThreadIDAndVersionLookup.java +++ b/core/src/main/java/org/elasticsearch/common/lucene/uid/PerThreadIDAndVersionLookup.java @@ -49,8 +49,6 @@ final class PerThreadIDAndVersionLookup { /** terms enum for uid field */ private final TermsEnum termsEnum; - /** _version data */ - private final NumericDocValues versions; /** Reused for iteration (when the term exists) */ private PostingsEnum docsEnum; @@ -59,7 +57,6 @@ final class PerThreadIDAndVersionLookup { */ PerThreadIDAndVersionLookup(LeafReader reader) throws IOException { TermsEnum termsEnum = null; - NumericDocValues versions = null; Fields fields = reader.fields(); if (fields != null) { @@ -67,12 +64,10 @@ final class PerThreadIDAndVersionLookup { if (terms != null) { termsEnum = terms.iterator(); assert termsEnum != null; - versions = reader.getNumericDocValues(VersionFieldMapper.NAME); - assert versions != null; + assert reader.getNumericDocValues(VersionFieldMapper.NAME) != null; } } - this.versions = versions; this.termsEnum = termsEnum; } @@ -90,7 +85,14 @@ public DocIdAndVersion lookup(BytesRef id, Bits liveDocs, LeafReaderContext cont } if (docID != DocIdSetIterator.NO_MORE_DOCS) { - return new DocIdAndVersion(docID, versions.get(docID), context); + final NumericDocValues versions = context.reader().getNumericDocValues(VersionFieldMapper.NAME); + if (versions == null) { + throw new AssertionError("Segment misses the _version field: " + context.reader()); + } + if (versions.advanceExact(docID) == false) { + throw new AssertionError("Document misses the _version field: " + docID + " in segment " + context.reader()); + } + return new DocIdAndVersion(docID, versions.longValue(), context); } } diff --git a/core/src/main/java/org/elasticsearch/index/query/functionscore/DecayFunctionBuilder.java b/core/src/main/java/org/elasticsearch/index/query/functionscore/DecayFunctionBuilder.java index 77b67b89fce0e..92b4e592165a6 100644 --- a/core/src/main/java/org/elasticsearch/index/query/functionscore/DecayFunctionBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/query/functionscore/DecayFunctionBuilder.java @@ -348,18 +348,18 @@ protected NumericDoubleValues distance(LeafReaderContext context) { final MultiGeoPointValues geoPointValues = fieldData.load(context).getGeoPointValues(); return mode.select(new MultiValueMode.UnsortedNumericDoubleValues() { @Override - public int count() { - return geoPointValues.count(); + public int docValueCount() { + return geoPointValues.docValueCount(); } @Override - public void setDocument(int docId) { - geoPointValues.setDocument(docId); + public boolean advanceExact(int docId) throws IOException { + return geoPointValues.advanceExact(docId); } @Override - public double valueAt(int index) { - GeoPoint other = geoPointValues.valueAt(index); + public double nextValue() { + GeoPoint other = geoPointValues.nextValue(); return Math.max(0.0d, distFunction.calculate(origin.lat(), origin.lon(), other.lat(), other.lon(), DistanceUnit.METERS) - offset); } @@ -367,15 +367,14 @@ public double valueAt(int index) { } @Override - protected String getDistanceString(LeafReaderContext ctx, int docId) { + protected String getDistanceString(LeafReaderContext ctx, int docId) throws IOException { StringBuilder values = new StringBuilder(mode.name()); values.append(" of: ["); final MultiGeoPointValues geoPointValues = fieldData.load(ctx).getGeoPointValues(); - geoPointValues.setDocument(docId); - final int num = geoPointValues.count(); - if (num > 0) { + if (geoPointValues.advanceExact(docId)) { + final int num = geoPointValues.docValueCount(); for (int i = 0; i < num; i++) { - GeoPoint value = geoPointValues.valueAt(i); + GeoPoint value = geoPointValues.nextValue(); values.append("Math.max(arcDistance("); values.append(value).append("(=doc value),"); values.append(origin).append("(=origin)) - ").append(offset).append("(=offset), 0)"); @@ -430,33 +429,32 @@ protected NumericDoubleValues distance(LeafReaderContext context) { final SortedNumericDoubleValues doubleValues = fieldData.load(context).getDoubleValues(); return mode.select(new MultiValueMode.UnsortedNumericDoubleValues() { @Override - public int count() { - return doubleValues.count(); + public int docValueCount() { + return doubleValues.docValueCount(); } @Override - public void setDocument(int docId) { - doubleValues.setDocument(docId); + public boolean advanceExact(int doc) throws IOException { + return doubleValues.advanceExact(doc); } @Override - public double valueAt(int index) { - return Math.max(0.0d, Math.abs(doubleValues.valueAt(index) - origin) - offset); + public double nextValue() throws IOException { + return Math.max(0.0d, Math.abs(doubleValues.nextValue() - origin) - offset); } }, 0.0); } @Override - protected String getDistanceString(LeafReaderContext ctx, int docId) { + protected String getDistanceString(LeafReaderContext ctx, int docId) throws IOException { StringBuilder values = new StringBuilder(mode.name()); values.append("["); final SortedNumericDoubleValues doubleValues = fieldData.load(ctx).getDoubleValues(); - doubleValues.setDocument(docId); - final int num = doubleValues.count(); - if (num > 0) { + if (doubleValues.advanceExact(docId)) { + final int num = doubleValues.docValueCount(); for (int i = 0; i < num; i++) { - double value = doubleValues.valueAt(i); + double value = doubleValues.nextValue(); values.append("Math.max(Math.abs("); values.append(value).append("(=doc value) - "); values.append(origin).append("(=origin))) - "); @@ -531,21 +529,28 @@ public final LeafScoreFunction getLeafScoreFunction(final LeafReaderContext ctx) return new LeafScoreFunction() { @Override - public double score(int docId, float subQueryScore) { - return func.evaluate(distance.get(docId), scale); + public double score(int docId, float subQueryScore) throws IOException { + if (distance.advanceExact(docId)) { + return func.evaluate(distance.doubleValue(), scale); + } else { + return 0; + } } @Override public Explanation explainScore(int docId, Explanation subQueryScore) throws IOException { + if (distance.advanceExact(docId) == false) { + return Explanation.noMatch("No value for the distance"); + } return Explanation.match( CombineFunction.toFloat(score(docId, subQueryScore.getValue())), "Function for field " + getFieldName() + ":", - func.explainFunction(getDistanceString(ctx, docId), distance.get(docId), scale)); + func.explainFunction(getDistanceString(ctx, docId), distance.doubleValue(), scale)); } }; } - protected abstract String getDistanceString(LeafReaderContext ctx, int docId); + protected abstract String getDistanceString(LeafReaderContext ctx, int docId) throws IOException; protected abstract String getFieldName(); From 7862e5f8569077c22d7dba9f31074775dbb6d474 Mon Sep 17 00:00:00 2001 From: Colin Goodheart-Smithe Date: Wed, 5 Apr 2017 16:30:54 +0100 Subject: [PATCH 18/87] Fixes compile errors on all remaining aggregations --- .../fielddata/AbstractSortedSetDocValues.java | 56 ++++++++ .../children/ParentToChildrenAggregator.java | 47 ++++--- .../geogrid/GeoGridAggregationBuilder.java | 23 ++-- .../bucket/range/BinaryRangeAggregator.java | 24 ++-- ...DiversifiedBytesHashSamplerAggregator.java | 30 +++-- .../DiversifiedMapSamplerAggregator.java | 36 +++--- .../DiversifiedNumericSamplerAggregator.java | 27 ++-- .../DiversifiedOrdinalsSamplerAggregator.java | 49 ++++--- ...balOrdinalsSignificantTermsAggregator.java | 22 ++-- .../GlobalOrdinalsStringTermsAggregator.java | 122 ++++++++++-------- .../bucket/terms/LongTermsAggregator.java | 46 +++---- .../bucket/terms/StringTermsAggregator.java | 55 ++++---- .../bucket/terms/support/IncludeExclude.java | 13 +- .../geocentroid/GeoCentroidAggregator.java | 10 +- .../geocentroid/InternalGeoCentroid.java | 18 ++- 15 files changed, 359 insertions(+), 219 deletions(-) create mode 100644 core/src/main/java/org/elasticsearch/index/fielddata/AbstractSortedSetDocValues.java diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/AbstractSortedSetDocValues.java b/core/src/main/java/org/elasticsearch/index/fielddata/AbstractSortedSetDocValues.java new file mode 100644 index 0000000000000..fedf815f26149 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/index/fielddata/AbstractSortedSetDocValues.java @@ -0,0 +1,56 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.fielddata; + +import org.apache.lucene.index.SortedSetDocValues; +import org.apache.lucene.search.DocIdSetIterator; + +import java.io.IOException; + +/** + * Base implementation that throws an {@link IOException} for the + * {@link DocIdSetIterator} APIs. This impl is safe to use for sorting and + * aggregations, which only use {@link #advanceExact(int)} and + * {@link #getValueCount()} and {@link #nextOrd()} and {@link #lookupOrd(long)}. + */ +public abstract class AbstractSortedSetDocValues extends SortedSetDocValues { + + @Override + public int docID() { + throw new UnsupportedOperationException(); + } + + @Override + public int nextDoc() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public int advance(int target) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public long cost() { + throw new UnsupportedOperationException(); + } + + +} diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/children/ParentToChildrenAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/children/ParentToChildrenAggregator.java index 5aa55c771b27c..37a443e9bab34 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/children/ParentToChildrenAggregator.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/children/ParentToChildrenAggregator.java @@ -57,20 +57,25 @@ public class ParentToChildrenAggregator extends SingleBucketAggregator { private final Weight parentFilter; private final ValuesSource.Bytes.WithOrdinals.ParentChild valuesSource; - // Maybe use PagedGrowableWriter? This will be less wasteful than LongArray, but then we don't have the reuse feature of BigArrays. - // Also if we know the highest possible value that a parent agg will create then we store multiple values into one slot + // Maybe use PagedGrowableWriter? This will be less wasteful than LongArray, + // but then we don't have the reuse feature of BigArrays. + // Also if we know the highest possible value that a parent agg will create + // then we store multiple values into one slot private final LongArray parentOrdToBuckets; // Only pay the extra storage price if the a parentOrd has multiple buckets - // Most of the times a parent doesn't have multiple buckets, since there is only one document per parent ord, - // only in the case of terms agg if a parent doc has multiple terms per field this is needed: + // Most of the times a parent doesn't have multiple buckets, since there is + // only one document per parent ord, + // only in the case of terms agg if a parent doc has multiple terms per + // field this is needed: private final LongObjectPagedHashMap parentOrdToOtherBuckets; private boolean multipleBucketsPerParentOrd = false; - public ParentToChildrenAggregator(String name, AggregatorFactories factories, SearchContext context, - Aggregator parent, String parentType, Query childFilter, Query parentFilter, - ValuesSource.Bytes.WithOrdinals.ParentChild valuesSource, - long maxOrd, List pipelineAggregators, Map metaData) throws IOException { + public ParentToChildrenAggregator(String name, AggregatorFactories factories, + SearchContext context, Aggregator parent, String parentType, Query childFilter, + Query parentFilter, ValuesSource.Bytes.WithOrdinals.ParentChild valuesSource, + long maxOrd, List pipelineAggregators, Map metaData) + throws IOException { super(name, factories, context, parent, pipelineAggregators, metaData); this.parentType = parentType; // these two filters are cached in the parser @@ -84,13 +89,14 @@ public ParentToChildrenAggregator(String name, AggregatorFactories factories, Se @Override public InternalAggregation buildAggregation(long owningBucketOrdinal) throws IOException { - return new InternalChildren(name, bucketDocCount(owningBucketOrdinal), bucketAggregations(owningBucketOrdinal), pipelineAggregators(), - metaData()); + return new InternalChildren(name, bucketDocCount(owningBucketOrdinal), + bucketAggregations(owningBucketOrdinal), pipelineAggregators(), metaData()); } @Override public InternalAggregation buildEmptyAggregation() { - return new InternalChildren(name, 0, buildEmptySubAggregations(), pipelineAggregators(), metaData()); + return new InternalChildren(name, 0, buildEmptySubAggregations(), pipelineAggregators(), + metaData()); } @Override @@ -108,8 +114,8 @@ public LeafBucketCollector getLeafCollector(LeafReaderContext ctx, @Override public void collect(int docId, long bucket) throws IOException { - if (parentDocs.get(docId)) { - long globalOrdinal = globalOrdinals.getOrd(docId); + if (parentDocs.get(docId) && globalOrdinals.advanceExact(docId)) { + long globalOrdinal = globalOrdinals.ordValue(); if (globalOrdinal != -1) { if (parentOrdToBuckets.get(globalOrdinal) == -1) { parentOrdToBuckets.set(globalOrdinal, bucket); @@ -120,7 +126,7 @@ public void collect(int docId, long bucket) throws IOException { bucketOrds[bucketOrds.length - 1] = bucket; parentOrdToOtherBuckets.put(globalOrdinal, bucketOrds); } else { - parentOrdToOtherBuckets.put(globalOrdinal, new long[]{bucket}); + parentOrdToOtherBuckets.put(globalOrdinal, new long[] { bucket }); } multipleBucketsPerParentOrd = true; } @@ -141,18 +147,21 @@ protected void doPostCollection() throws IOException { DocIdSetIterator childDocsIter = childDocsScorer.iterator(); final LeafBucketCollector sub = collectableSubAggregators.getLeafCollector(ctx); - final SortedDocValues globalOrdinals = valuesSource.globalOrdinalsValues(parentType, ctx); + final SortedDocValues globalOrdinals = valuesSource.globalOrdinalsValues(parentType, + ctx); // Set the scorer, since we now replay only the child docIds - sub.setScorer(new ConstantScoreScorer(null, 1f,childDocsIter)); + sub.setScorer(new ConstantScoreScorer(null, 1f, childDocsIter)); final Bits liveDocs = ctx.reader().getLiveDocs(); - for (int docId = childDocsIter.nextDoc(); docId != DocIdSetIterator.NO_MORE_DOCS; docId = childDocsIter.nextDoc()) { + for (int docId = childDocsIter + .nextDoc(); docId != DocIdSetIterator.NO_MORE_DOCS; docId = childDocsIter + .nextDoc()) { if (liveDocs != null && liveDocs.get(docId) == false) { continue; } - long globalOrdinal = globalOrdinals.getOrd(docId); - if (globalOrdinal != -1) { + if (globalOrdinals.advanceExact(docId)) { + long globalOrdinal = globalOrdinals.ordValue(); long bucketOrd = parentOrdToBuckets.get(globalOrdinal); if (bucketOrd != -1) { collectBucket(sub, docId, bucketOrd); diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregationBuilder.java index 602c3a81c66e9..1c60247446562 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregationBuilder.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregationBuilder.java @@ -28,10 +28,10 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.ObjectParser; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.index.fielddata.AbstractSortingNumericDocValues; import org.elasticsearch.index.fielddata.MultiGeoPointValues; import org.elasticsearch.index.fielddata.SortedBinaryDocValues; import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; -import org.elasticsearch.index.fielddata.SortingNumericDocValues; import org.elasticsearch.index.query.QueryParseContext; import org.elasticsearch.search.aggregations.AggregatorFactories.Builder; import org.elasticsearch.search.aggregations.AggregatorFactory; @@ -187,7 +187,7 @@ public String getType() { return NAME; } - private static class CellValues extends SortingNumericDocValues { + private static class CellValues extends AbstractSortingNumericDocValues { private MultiGeoPointValues geoValues; private int precision; @@ -197,14 +197,19 @@ protected CellValues(MultiGeoPointValues geoValues, int precision) { } @Override - public void setDocument(int docId) { - geoValues.setDocument(docId); - resize(geoValues.count()); - for (int i = 0; i < count(); ++i) { - GeoPoint target = geoValues.valueAt(i); - values[i] = GeoHashUtils.longEncode(target.getLon(), target.getLat(), precision); + public boolean advanceExact(int docId) throws IOException { + if (geoValues.advanceExact(docId)) { + resize(geoValues.docValueCount()); + for (int i = 0; i < docValueCount(); ++i) { + GeoPoint target = geoValues.nextValue(); + values[i] = GeoHashUtils.longEncode(target.getLon(), target.getLat(), + precision); + } + sort(); + return true; + } else { + return false; } - sort(); } } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/BinaryRangeAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/BinaryRangeAggregator.java index 873eef2f71566..9151ebc985adb 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/BinaryRangeAggregator.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/BinaryRangeAggregator.java @@ -127,7 +127,7 @@ abstract static class SortedSetRangeLeafCollector extends LeafBucketCollectorBas final LeafBucketCollector sub; SortedSetRangeLeafCollector(SortedSetDocValues values, - Range[] ranges, LeafBucketCollector sub) { + Range[] ranges, LeafBucketCollector sub) throws IOException { super(sub, values); for (int i = 1; i < ranges.length; ++i) { if (RANGE_COMPARATOR.compare(ranges[i-1], ranges[i]) > 0) { @@ -167,10 +167,13 @@ abstract static class SortedSetRangeLeafCollector extends LeafBucketCollectorBas @Override public void collect(int doc, long bucket) throws IOException { - values.setDocument(doc); - int lo = 0; - for (long ord = values.nextOrd(); ord != SortedSetDocValues.NO_MORE_ORDS; ord = values.nextOrd()) { - lo = collect(doc, ord, bucket, lo); + if (values.advanceExact(doc)) { + int lo = 0; + for (long ord = values + .nextOrd(); ord != SortedSetDocValues.NO_MORE_ORDS; ord = values + .nextOrd()) { + lo = collect(doc, ord, bucket, lo); + } } } @@ -259,11 +262,12 @@ abstract static class SortedBinaryRangeLeafCollector extends LeafBucketCollector @Override public void collect(int doc, long bucket) throws IOException { - values.setDocument(doc); - final int valuesCount = values.count(); - for (int i = 0, lo = 0; i < valuesCount; ++i) { - final BytesRef value = values.valueAt(i); - lo = collect(doc, value, bucket, lo); + if (values.advanceExact(doc)) { + final int valuesCount = values.docValueCount(); + for (int i = 0, lo = 0; i < valuesCount; ++i) { + final BytesRef value = values.nextValue(); + lo = collect(doc, value, bucket, lo); + } } } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedBytesHashSamplerAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedBytesHashSamplerAggregator.java index 33532a9f43846..0bc678d25e4e8 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedBytesHashSamplerAggregator.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedBytesHashSamplerAggregator.java @@ -26,6 +26,7 @@ import org.apache.lucene.search.TopDocsCollector; import org.apache.lucene.util.BytesRef; import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.index.fielddata.AbstractNumericDocValues; import org.elasticsearch.index.fielddata.SortedBinaryDocValues; import org.elasticsearch.search.aggregations.Aggregator; import org.elasticsearch.search.aggregations.AggregatorFactories; @@ -97,20 +98,25 @@ protected NumericDocValues getKeys(LeafReaderContext context) { } catch (IOException e) { throw new ElasticsearchException("Error reading values", e); } - return new NumericDocValues() { - @Override - public long get(int doc) { + return new AbstractNumericDocValues() { - values.setDocument(doc); - final int valuesCount = values.count(); - if (valuesCount > 1) { - throw new IllegalArgumentException("Sample diversifying key must be a single valued-field"); - } - if (valuesCount == 1) { - final BytesRef bytes = values.valueAt(0); - return bytes.hashCode(); + @Override + public boolean advanceExact(int target) throws IOException { + if (values.advanceExact(target)) { + if (values.docValueCount() > 1) { + throw new IllegalArgumentException( + "Sample diversifying key must be a single valued-field"); + } + return true; + } else { + return false; } - return 0; + } + + @Override + public long longValue() throws IOException { + final BytesRef bytes = values.nextValue(); + return bytes.hashCode(); } }; } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedMapSamplerAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedMapSamplerAggregator.java index d16e798b9614d..0480e013c10fe 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedMapSamplerAggregator.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedMapSamplerAggregator.java @@ -28,6 +28,7 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.common.lease.Releasables; import org.elasticsearch.common.util.BytesRefHash; +import org.elasticsearch.index.fielddata.AbstractNumericDocValues; import org.elasticsearch.index.fielddata.SortedBinaryDocValues; import org.elasticsearch.search.aggregations.Aggregator; import org.elasticsearch.search.aggregations.AggregatorFactories; @@ -103,25 +104,30 @@ protected NumericDocValues getKeys(LeafReaderContext context) { } catch (IOException e) { throw new ElasticsearchException("Error reading values", e); } - return new NumericDocValues() { - @Override - public long get(int doc) { + return new AbstractNumericDocValues() { - values.setDocument(doc); - final int valuesCount = values.count(); - if (valuesCount > 1) { - throw new IllegalArgumentException("Sample diversifying key must be a single valued-field"); + @Override + public boolean advanceExact(int target) throws IOException { + if (values.advanceExact(target)) { + if (values.docValueCount() > 1) { + throw new IllegalArgumentException( + "Sample diversifying key must be a single valued-field"); + } + return true; + } else { + return false; } - if (valuesCount == 1) { - final BytesRef bytes = values.valueAt(0); + } - long bucketOrdinal = bucketOrds.add(bytes); - if (bucketOrdinal < 0) { // already seen - bucketOrdinal = -1 - bucketOrdinal; - } - return bucketOrdinal; + @Override + public long longValue() throws IOException { + final BytesRef bytes = values.nextValue(); + + long bucketOrdinal = bucketOrds.add(bytes); + if (bucketOrdinal < 0) { // already seen + bucketOrdinal = -1 - bucketOrdinal; } - return 0; + return bucketOrdinal; } }; } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedNumericSamplerAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedNumericSamplerAggregator.java index c33a1a5553244..4badde70b5b1d 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedNumericSamplerAggregator.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedNumericSamplerAggregator.java @@ -26,6 +26,7 @@ import org.apache.lucene.search.DiversifiedTopDocsCollector.ScoreDocKey; import org.apache.lucene.search.TopDocsCollector; import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.index.fielddata.AbstractNumericDocValues; import org.elasticsearch.search.aggregations.Aggregator; import org.elasticsearch.search.aggregations.AggregatorFactories; import org.elasticsearch.search.aggregations.bucket.DeferringBucketCollector; @@ -88,18 +89,24 @@ protected NumericDocValues getKeys(LeafReaderContext context) { } catch (IOException e) { throw new ElasticsearchException("Error reading values", e); } - return new NumericDocValues() { + return new AbstractNumericDocValues() { + @Override - public long get(int doc) { - values.setDocument(doc); - final int valuesCount = values.count(); - if (valuesCount > 1) { - throw new IllegalArgumentException("Sample diversifying key must be a single valued-field"); - } - if (valuesCount == 1) { - return values.valueAt(0); + public boolean advanceExact(int target) throws IOException { + if (values.advanceExact(target)) { + if (values.docValueCount() > 1) { + throw new IllegalArgumentException( + "Sample diversifying key must be a single valued-field"); + } + return true; + } else { + return false; } - return Long.MIN_VALUE; + } + + @Override + public long longValue() throws IOException { + return values.nextValue(); } }; } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedOrdinalsSamplerAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedOrdinalsSamplerAggregator.java index a5d027debe1b0..cda371c6e734b 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedOrdinalsSamplerAggregator.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedOrdinalsSamplerAggregator.java @@ -22,11 +22,12 @@ import org.apache.lucene.index.DocValues; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.NumericDocValues; -import org.apache.lucene.index.RandomAccessOrds; import org.apache.lucene.index.SortedDocValues; +import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.search.DiversifiedTopDocsCollector; import org.apache.lucene.search.DiversifiedTopDocsCollector.ScoreDocKey; import org.apache.lucene.search.TopDocsCollector; +import org.elasticsearch.index.fielddata.AbstractNumericDocValues; import org.elasticsearch.search.aggregations.Aggregator; import org.elasticsearch.search.aggregations.AggregatorFactories; import org.elasticsearch.search.aggregations.bucket.DeferringBucketCollector; @@ -84,29 +85,45 @@ class ValuesDiversifiedTopDocsCollector extends DiversifiedTopDocsCollector { @Override protected NumericDocValues getKeys(LeafReaderContext context) { - final RandomAccessOrds globalOrds = valuesSource.globalOrdinalsValues(context); + final SortedSetDocValues globalOrds = valuesSource.globalOrdinalsValues(context); final SortedDocValues singleValues = DocValues.unwrapSingleton(globalOrds); if (singleValues != null) { - return new NumericDocValues() { + return new AbstractNumericDocValues() { + @Override - public long get(int doc) { - return singleValues.getOrd(doc); + public boolean advanceExact(int target) throws IOException { + return singleValues.advanceExact(target); + } + + @Override + public long longValue() throws IOException { + return singleValues.ordValue(); } }; } - return new NumericDocValues() { + return new AbstractNumericDocValues() { + + long value; + @Override - public long get(int doc) { - globalOrds.setDocument(doc); - final long valuesCount = globalOrds.cardinality(); - if (valuesCount > 1) { - throw new IllegalArgumentException("Sample diversifying key must be a single valued-field"); - } - if (valuesCount == 1) { - long result = globalOrds.ordAt(0); - return result; + public boolean advanceExact(int target) throws IOException { + if (globalOrds.advanceExact(target)) { + value = globalOrds.nextOrd(); + // Check there isn't a second value for this + // document + if (globalOrds.nextOrd() != SortedSetDocValues.NO_MORE_ORDS) { + throw new IllegalArgumentException( + "Sample diversifying key must be a single valued-field"); + } + return true; + } else { + return false; } - return Long.MIN_VALUE; + } + + @Override + public long longValue() throws IOException { + return value; } }; diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/GlobalOrdinalsSignificantTermsAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/GlobalOrdinalsSignificantTermsAggregator.java index bf1714b14cc23..98effdcfd54c8 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/GlobalOrdinalsSignificantTermsAggregator.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/GlobalOrdinalsSignificantTermsAggregator.java @@ -20,6 +20,7 @@ import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.lease.Releasables; import org.elasticsearch.common.util.LongHash; @@ -177,16 +178,17 @@ public LeafBucketCollector getLeafCollector(LeafReaderContext ctx, public void collect(int doc, long bucket) throws IOException { assert bucket == 0; numCollectedDocs++; - globalOrds.setDocument(doc); - final int numOrds = globalOrds.cardinality(); - for (int i = 0; i < numOrds; i++) { - final long globalOrd = globalOrds.ordAt(i); - long bucketOrd = bucketOrds.add(globalOrd); - if (bucketOrd < 0) { - bucketOrd = -1 - bucketOrd; - collectExistingBucket(sub, doc, bucketOrd); - } else { - collectBucket(sub, doc, bucketOrd); + if (globalOrds.advanceExact(doc)) { + for (long globalOrd = globalOrds.nextOrd(); + globalOrd != SortedSetDocValues.NO_MORE_ORDS; + globalOrd = globalOrds.nextOrd()) { + long bucketOrd = bucketOrds.add(globalOrd); + if (bucketOrd < 0) { + bucketOrd = -1 - bucketOrd; + collectExistingBucket(sub, doc, bucketOrd); + } else { + collectBucket(sub, doc, bucketOrd); + } } } } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/GlobalOrdinalsStringTermsAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/GlobalOrdinalsStringTermsAggregator.java index 8e9d779cd08a1..cb00bdf93f222 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/GlobalOrdinalsStringTermsAggregator.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/GlobalOrdinalsStringTermsAggregator.java @@ -21,8 +21,8 @@ import org.apache.lucene.index.DocValues; import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.RandomAccessOrds; import org.apache.lucene.index.SortedDocValues; +import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.LongBitSet; @@ -31,7 +31,7 @@ import org.elasticsearch.common.util.IntArray; import org.elasticsearch.common.util.LongHash; import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.index.fielddata.AbstractRandomAccessOrds; +import org.elasticsearch.index.fielddata.AbstractSortedSetDocValues; import org.elasticsearch.index.fielddata.ordinals.GlobalOrdinalMapping; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.aggregations.Aggregator; @@ -67,7 +67,7 @@ public class GlobalOrdinalsStringTermsAggregator extends AbstractStringTermsAggr // especially if this agg is on a second layer or deeper. protected LongBitSet acceptedGlobalOrdinals; - protected RandomAccessOrds globalOrds; + protected SortedSetDocValues globalOrds; public GlobalOrdinalsStringTermsAggregator(String name, AggregatorFactories factories, ValuesSource.Bytes.WithOrdinals valuesSource, Terms.Order order, DocValueFormat format, BucketCountThresholds bucketCountThresholds, @@ -101,7 +101,8 @@ public LeafBucketCollector getLeafCollector(LeafReaderContext ctx, return newCollector(globalOrds, sub); } - protected LeafBucketCollector newCollector(final RandomAccessOrds ords, final LeafBucketCollector sub) { + protected LeafBucketCollector newCollector(final SortedSetDocValues ords, + final LeafBucketCollector sub) { grow(ords.getValueCount()); final SortedDocValues singleValues = DocValues.unwrapSingleton(ords); if (singleValues != null) { @@ -109,8 +110,8 @@ protected LeafBucketCollector newCollector(final RandomAccessOrds ords, final Le @Override public void collect(int doc, long bucket) throws IOException { assert bucket == 0; - final int ord = singleValues.getOrd(doc); - if (ord >= 0) { + if (singleValues.advanceExact(doc)) { + final int ord = singleValues.ordValue(); collectExistingBucket(sub, doc, ord); } } @@ -120,11 +121,12 @@ public void collect(int doc, long bucket) throws IOException { @Override public void collect(int doc, long bucket) throws IOException { assert bucket == 0; - ords.setDocument(doc); - final int numOrds = ords.cardinality(); - for (int i = 0; i < numOrds; i++) { - final long globalOrd = ords.ordAt(i); - collectExistingBucket(sub, doc, globalOrd); + if (ords.advanceExact(doc)) { + for (long globalOrd = ords.nextOrd(); + globalOrd != SortedSetDocValues.NO_MORE_ORDS; + globalOrd = ords.nextOrd()) { + collectExistingBucket(sub, doc, globalOrd); + } } } }; @@ -270,14 +272,15 @@ public WithHash(String name, AggregatorFactories factories, ValuesSource.Bytes.W } @Override - protected LeafBucketCollector newCollector(final RandomAccessOrds ords, final LeafBucketCollector sub) { + protected LeafBucketCollector newCollector(final SortedSetDocValues ords, + final LeafBucketCollector sub) { final SortedDocValues singleValues = DocValues.unwrapSingleton(ords); if (singleValues != null) { return new LeafBucketCollectorBase(sub, ords) { @Override public void collect(int doc, long bucket) throws IOException { - final int globalOrd = singleValues.getOrd(doc); - if (globalOrd >= 0) { + if (singleValues.advanceExact(doc)) { + final int globalOrd = singleValues.ordValue(); long bucketOrd = bucketOrds.add(globalOrd); if (bucketOrd < 0) { bucketOrd = -1 - bucketOrd; @@ -292,16 +295,17 @@ public void collect(int doc, long bucket) throws IOException { return new LeafBucketCollectorBase(sub, ords) { @Override public void collect(int doc, long bucket) throws IOException { - ords.setDocument(doc); - final int numOrds = ords.cardinality(); - for (int i = 0; i < numOrds; i++) { - final long globalOrd = ords.ordAt(i); - long bucketOrd = bucketOrds.add(globalOrd); - if (bucketOrd < 0) { - bucketOrd = -1 - bucketOrd; - collectExistingBucket(sub, doc, bucketOrd); - } else { - collectBucket(sub, doc, bucketOrd); + if (ords.advanceExact(doc)) { + for (long globalOrd = ords.nextOrd(); + globalOrd != SortedSetDocValues.NO_MORE_ORDS; + globalOrd = ords.nextOrd()) { + long bucketOrd = bucketOrds.add(globalOrd); + if (bucketOrd < 0) { + bucketOrd = -1 - bucketOrd; + collectExistingBucket(sub, doc, bucketOrd); + } else { + collectBucket(sub, doc, bucketOrd); + } } } } @@ -330,7 +334,7 @@ public static class LowCardinality extends GlobalOrdinalsStringTermsAggregator { private IntArray segmentDocCounts; - private RandomAccessOrds segmentOrds; + private SortedSetDocValues segmentOrds; public LowCardinality(String name, AggregatorFactories factories, ValuesSource.Bytes.WithOrdinals valuesSource, Terms.Order order, DocValueFormat format, @@ -345,7 +349,8 @@ public LowCardinality(String name, AggregatorFactories factories, ValuesSource.B // bucketOrd is ord + 1 to avoid a branch to deal with the missing ord @Override - protected LeafBucketCollector newCollector(final RandomAccessOrds ords, LeafBucketCollector sub) { + protected LeafBucketCollector newCollector(final SortedSetDocValues ords, + LeafBucketCollector sub) { segmentDocCounts = context.bigArrays().grow(segmentDocCounts, 1 + ords.getValueCount()); assert sub == LeafBucketCollector.NO_OP_COLLECTOR; final SortedDocValues singleValues = DocValues.unwrapSingleton(ords); @@ -354,8 +359,10 @@ protected LeafBucketCollector newCollector(final RandomAccessOrds ords, LeafBuck @Override public void collect(int doc, long bucket) throws IOException { assert bucket == 0; - final int ord = singleValues.getOrd(doc); - segmentDocCounts.increment(ord + 1, 1); + if (singleValues.advanceExact(doc)) { + final int ord = singleValues.ordValue(); + segmentDocCounts.increment(ord + 1, 1); + } } }; } else { @@ -363,11 +370,12 @@ public void collect(int doc, long bucket) throws IOException { @Override public void collect(int doc, long bucket) throws IOException { assert bucket == 0; - ords.setDocument(doc); - final int numOrds = ords.cardinality(); - for (int i = 0; i < numOrds; i++) { - final long segmentOrd = ords.ordAt(i); - segmentDocCounts.increment(segmentOrd + 1, 1); + if (ords.advanceExact(doc)) { + for (long segmentOrd = ords.nextOrd(); + segmentOrd != SortedSetDocValues.NO_MORE_ORDS; + segmentOrd = ords.nextOrd()) { + segmentDocCounts.increment(segmentOrd + 1, 1); + } } } }; @@ -422,15 +430,16 @@ private void mapSegmentCountsToGlobalCounts() { } } - private static final class FilteredOrdinals extends AbstractRandomAccessOrds { + private static final class FilteredOrdinals extends AbstractSortedSetDocValues { - private final RandomAccessOrds inner; + private final SortedSetDocValues inner; private final LongBitSet accepted; private int cardinality; private long[] ords = new long[0]; + private int ordsCursor = 0; - private FilteredOrdinals(RandomAccessOrds inner, LongBitSet accepted) { + private FilteredOrdinals(SortedSetDocValues inner, LongBitSet accepted) { this.inner = inner; this.accepted = accepted; } @@ -441,33 +450,32 @@ public long getValueCount() { } @Override - public long ordAt(int index) { - return ords[index]; - } - - @Override - public void doSetDocument(int docId) { - inner.setDocument(docId); - final int innerCardinality = inner.cardinality(); - ords = ArrayUtil.grow(ords, innerCardinality); - - cardinality = 0; - for (int slot = 0; slot < innerCardinality; slot++) { - long ord = inner.ordAt(slot); - if (accepted.get(ord)) { - ords[cardinality++] = ord; - } - } + public BytesRef lookupOrd(long ord) throws IOException { + return inner.lookupOrd(ord); } @Override - public int cardinality() { - return cardinality; + public long nextOrd() throws IOException { + return ords[ordsCursor++]; } @Override - public BytesRef lookupOrd(long ord) { - return inner.lookupOrd(ord); + public boolean advanceExact(int target) throws IOException { + if (inner.advanceExact(target)) { + ordsCursor = 0; + + cardinality = 0; + for (long ord = inner.nextOrd(); ord != SortedSetDocValues.NO_MORE_ORDS; + ord = inner.nextOrd()) { + if (accepted.get(ord)) { + ords = ArrayUtil.grow(ords, cardinality + 1); + ords[cardinality++] = ord; + } + } + return true; + } else { + return false; + } } } } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/LongTermsAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/LongTermsAggregator.java index e041ada417f0c..8f8e2f3079b67 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/LongTermsAggregator.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/LongTermsAggregator.java @@ -77,24 +77,25 @@ public LeafBucketCollector getLeafCollector(LeafReaderContext ctx, @Override public void collect(int doc, long owningBucketOrdinal) throws IOException { assert owningBucketOrdinal == 0; - values.setDocument(doc); - final int valuesCount = values.count(); - - long previous = Long.MAX_VALUE; - for (int i = 0; i < valuesCount; ++i) { - final long val = values.valueAt(i); - if (previous != val || i == 0) { - if ((longFilter == null) || (longFilter.accept(val))) { - long bucketOrdinal = bucketOrds.add(val); - if (bucketOrdinal < 0) { // already seen - bucketOrdinal = - 1 - bucketOrdinal; - collectExistingBucket(sub, doc, bucketOrdinal); - } else { - collectBucket(sub, doc, bucketOrdinal); + if (values.advanceExact(doc)) { + final int valuesCount = values.docValueCount(); + + long previous = Long.MAX_VALUE; + for (int i = 0; i < valuesCount; ++i) { + final long val = values.nextValue(); + if (previous != val || i == 0) { + if ((longFilter == null) || (longFilter.accept(val))) { + long bucketOrdinal = bucketOrds.add(val); + if (bucketOrdinal < 0) { // already seen + bucketOrdinal = -1 - bucketOrdinal; + collectExistingBucket(sub, doc, bucketOrdinal); + } else { + collectBucket(sub, doc, bucketOrdinal); + } } - } - previous = val; + previous = val; + } } } } @@ -110,12 +111,13 @@ public InternalAggregation buildAggregation(long owningBucketOrdinal) throws IOE for (LeafReaderContext ctx : context.searcher().getTopReaderContext().leaves()) { final SortedNumericDocValues values = getValues(valuesSource, ctx); for (int docId = 0; docId < ctx.reader().maxDoc(); ++docId) { - values.setDocument(docId); - final int valueCount = values.count(); - for (int i = 0; i < valueCount; ++i) { - long value = values.valueAt(i); - if (longFilter == null || longFilter.accept(value)) { - bucketOrds.add(value); + if (values.advanceExact(docId)) { + final int valueCount = values.docValueCount(); + for (int i = 0; i < valueCount; ++i) { + long value = values.nextValue(); + if (longFilter == null || longFilter.accept(value)) { + bucketOrds.add(value); + } } } } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/StringTermsAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/StringTermsAggregator.java index c93fc94ff6187..61c46cdfd68a9 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/StringTermsAggregator.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/StringTermsAggregator.java @@ -78,27 +78,29 @@ public LeafBucketCollector getLeafCollector(LeafReaderContext ctx, @Override public void collect(int doc, long bucket) throws IOException { assert bucket == 0; - values.setDocument(doc); - final int valuesCount = values.count(); - - // SortedBinaryDocValues don't guarantee uniqueness so we need to take care of dups - previous.clear(); - for (int i = 0; i < valuesCount; ++i) { - final BytesRef bytes = values.valueAt(i); - if (includeExclude != null && !includeExclude.accept(bytes)) { - continue; - } - if (previous.get().equals(bytes)) { - continue; - } - long bucketOrdinal = bucketOrds.add(bytes); - if (bucketOrdinal < 0) { // already seen - bucketOrdinal = - 1 - bucketOrdinal; - collectExistingBucket(sub, doc, bucketOrdinal); - } else { - collectBucket(sub, doc, bucketOrdinal); + if (values.advanceExact(doc)) { + final int valuesCount = values.docValueCount(); + + // SortedBinaryDocValues don't guarantee uniqueness so we + // need to take care of dups + previous.clear(); + for (int i = 0; i < valuesCount; ++i) { + final BytesRef bytes = values.nextValue(); + if (includeExclude != null && !includeExclude.accept(bytes)) { + continue; + } + if (previous.get().equals(bytes)) { + continue; + } + long bucketOrdinal = bucketOrds.add(bytes); + if (bucketOrdinal < 0) { // already seen + bucketOrdinal = -1 - bucketOrdinal; + collectExistingBucket(sub, doc, bucketOrdinal); + } else { + collectBucket(sub, doc, bucketOrdinal); + } + previous.copyBytes(bytes); } - previous.copyBytes(bytes); } } }; @@ -114,12 +116,13 @@ public InternalAggregation buildAggregation(long owningBucketOrdinal) throws IOE final SortedBinaryDocValues values = valuesSource.bytesValues(ctx); // brute force for (int docId = 0; docId < ctx.reader().maxDoc(); ++docId) { - values.setDocument(docId); - final int valueCount = values.count(); - for (int i = 0; i < valueCount; ++i) { - final BytesRef term = values.valueAt(i); - if (includeExclude == null || includeExclude.accept(term)) { - bucketOrds.add(term); + if (values.advanceExact(docId)) { + final int valueCount = values.docValueCount(); + for (int i = 0; i < valueCount; ++i) { + final BytesRef term = values.nextValue(); + if (includeExclude == null || includeExclude.accept(term)) { + bucketOrds.add(term); + } } } } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/support/IncludeExclude.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/support/IncludeExclude.java index 15207102108b7..aabe5f585da81 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/support/IncludeExclude.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/support/IncludeExclude.java @@ -21,7 +21,7 @@ import com.carrotsearch.hppc.BitMixer; import com.carrotsearch.hppc.LongHashSet; import com.carrotsearch.hppc.LongSet; -import org.apache.lucene.index.RandomAccessOrds; + import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; @@ -233,14 +233,16 @@ public boolean accept(BytesRef value) { } public abstract static class OrdinalsFilter { - public abstract LongBitSet acceptedGlobalOrdinals(RandomAccessOrds globalOrdinals) throws IOException; + public abstract LongBitSet acceptedGlobalOrdinals(SortedSetDocValues globalOrdinals) + throws IOException; } class PartitionedOrdinalsFilter extends OrdinalsFilter { @Override - public LongBitSet acceptedGlobalOrdinals(RandomAccessOrds globalOrdinals) throws IOException { + public LongBitSet acceptedGlobalOrdinals(SortedSetDocValues globalOrdinals) + throws IOException { final long numOrds = globalOrdinals.getValueCount(); final LongBitSet acceptedGlobalOrdinals = new LongBitSet(numOrds); final TermsEnum termEnum = globalOrdinals.termsEnum(); @@ -269,7 +271,7 @@ private AutomatonBackedOrdinalsFilter(Automaton automaton) { * */ @Override - public LongBitSet acceptedGlobalOrdinals(RandomAccessOrds globalOrdinals) + public LongBitSet acceptedGlobalOrdinals(SortedSetDocValues globalOrdinals) throws IOException { LongBitSet acceptedGlobalOrdinals = new LongBitSet(globalOrdinals.getValueCount()); TermsEnum globalTermsEnum; @@ -295,7 +297,8 @@ static class TermListBackedOrdinalsFilter extends OrdinalsFilter { } @Override - public LongBitSet acceptedGlobalOrdinals(RandomAccessOrds globalOrdinals) throws IOException { + public LongBitSet acceptedGlobalOrdinals(SortedSetDocValues globalOrdinals) + throws IOException { LongBitSet acceptedGlobalOrdinals = new LongBitSet(globalOrdinals.getValueCount()); if (includeValues != null) { for (BytesRef term : includeValues) { diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/GeoCentroidAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/GeoCentroidAggregator.java index c6a5326a773b3..d14c624243046 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/GeoCentroidAggregator.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/GeoCentroidAggregator.java @@ -20,7 +20,6 @@ package org.elasticsearch.search.aggregations.metrics.geocentroid; import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.spatial.geopoint.document.GeoPointField; import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.common.lease.Releasables; import org.elasticsearch.common.util.BigArrays; @@ -83,8 +82,8 @@ public void collect(int doc, long bucket) throws IOException { // computed if (prevCounts > 0) { final long mortonCode = centroids.get(bucket); - pt[0] = GeoPointField.decodeLongitude(mortonCode); - pt[1] = GeoPointField.decodeLatitude(mortonCode); + pt[0] = InternalGeoCentroid.decodeLongitude(mortonCode); + pt[1] = InternalGeoCentroid.decodeLatitude(mortonCode); } // update the moving average for (int i = 0; i < valueCount; ++i) { @@ -95,7 +94,7 @@ public void collect(int doc, long bucket) throws IOException { // TODO: we do not need to interleave the lat and lon // bits here // should we just store them contiguously? - centroids.set(bucket, GeoPointField.encodeLatLon(pt[1], pt[0])); + centroids.set(bucket, InternalGeoCentroid.encodeLatLon(pt[1], pt[0])); } } }; @@ -109,7 +108,8 @@ public InternalAggregation buildAggregation(long bucket) { final long bucketCount = counts.get(bucket); final long mortonCode = centroids.get(bucket); final GeoPoint bucketCentroid = (bucketCount > 0) - ? new GeoPoint(GeoPointField.decodeLatitude(mortonCode), GeoPointField.decodeLongitude(mortonCode)) + ? new GeoPoint(InternalGeoCentroid.decodeLatitude(mortonCode), + InternalGeoCentroid.decodeLongitude(mortonCode)) : null; return new InternalGeoCentroid(name, bucketCentroid , bucketCount, pipelineAggregators(), metaData()); } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/InternalGeoCentroid.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/InternalGeoCentroid.java index a5a8058ed2835..f97423f3bd09b 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/InternalGeoCentroid.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/InternalGeoCentroid.java @@ -19,7 +19,7 @@ package org.elasticsearch.search.aggregations.metrics.geocentroid; -import org.apache.lucene.spatial.geopoint.document.GeoPointField; +import org.apache.lucene.geo.GeoEncodingUtils; import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -38,6 +38,18 @@ public class InternalGeoCentroid extends InternalAggregation implements GeoCentr protected final GeoPoint centroid; protected final long count; + public static long encodeLatLon(double lat, double lon) { + return (GeoEncodingUtils.encodeLatitude(lat) << 32) + GeoEncodingUtils.encodeLongitude(lon); + } + + public static double decodeLatitude(long encodedLatLon) { + return GeoEncodingUtils.decodeLatitude((int) (encodedLatLon >>> 32)); + } + + public static double decodeLongitude(long encodedLatLon) { + return GeoEncodingUtils.decodeLatitude((int) encodedLatLon); + } + public InternalGeoCentroid(String name, GeoPoint centroid, long count, List pipelineAggregators, Map metaData) { super(name, pipelineAggregators, metaData); @@ -55,7 +67,7 @@ public InternalGeoCentroid(StreamInput in) throws IOException { count = in.readVLong(); if (in.readBoolean()) { final long hash = in.readLong(); - centroid = new GeoPoint(GeoPointField.decodeLatitude(hash), GeoPointField.decodeLongitude(hash)); + centroid = new GeoPoint(decodeLatitude(hash), decodeLongitude(hash)); } else { centroid = null; } @@ -67,7 +79,7 @@ protected void doWriteTo(StreamOutput out) throws IOException { if (centroid != null) { out.writeBoolean(true); // should we just write lat and lon separately? - out.writeLong(GeoPointField.encodeLatLon(centroid.lat(), centroid.lon())); + out.writeLong(encodeLatLon(centroid.lat(), centroid.lon())); } else { out.writeBoolean(false); } From 3713bed1539e1f9438305d81db170853c452dca8 Mon Sep 17 00:00:00 2001 From: Colin Goodheart-Smithe Date: Wed, 5 Apr 2017 17:04:12 +0100 Subject: [PATCH 19/87] Fixes compile errors for function score and some fetch phases --- .../lucene/search/function/FieldValueFactorFunction.java | 8 +++++--- .../search/fetch/subphase/ParentFieldSubFetchPhase.java | 4 ++-- .../search/fetch/subphase/VersionFetchSubPhase.java | 4 ++-- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/common/lucene/search/function/FieldValueFactorFunction.java b/core/src/main/java/org/elasticsearch/common/lucene/search/function/FieldValueFactorFunction.java index 2c5fd5924597c..e225df040ab28 100644 --- a/core/src/main/java/org/elasticsearch/common/lucene/search/function/FieldValueFactorFunction.java +++ b/core/src/main/java/org/elasticsearch/common/lucene/search/function/FieldValueFactorFunction.java @@ -74,10 +74,12 @@ public double score(int docId, float subQueryScore) throws IOException { double value; if (values.advanceExact(docId)) { value = values.nextValue(); - } else if (missing != null) { - value = missing; } else { - throw new ElasticsearchException("Missing value for field [" + field + "]"); + if (missing != null) { + value = missing; + } else { + throw new ElasticsearchException("Missing value for field [" + field + "]"); + } } double val = value * boostFactor; double result = modifier.apply(val); diff --git a/core/src/main/java/org/elasticsearch/search/fetch/subphase/ParentFieldSubFetchPhase.java b/core/src/main/java/org/elasticsearch/search/fetch/subphase/ParentFieldSubFetchPhase.java index 00b96867026cc..7a0a530e621be 100644 --- a/core/src/main/java/org/elasticsearch/search/fetch/subphase/ParentFieldSubFetchPhase.java +++ b/core/src/main/java/org/elasticsearch/search/fetch/subphase/ParentFieldSubFetchPhase.java @@ -62,11 +62,11 @@ public void hitExecute(SearchContext context, HitContext hitContext) { public static String getParentId(ParentFieldMapper fieldMapper, LeafReader reader, int docId) { try { SortedDocValues docValues = reader.getSortedDocValues(fieldMapper.name()); - if (docValues == null) { + if (docValues == null || docValues.advanceExact(docId)) { // hit has no _parent field. return null; } - BytesRef parentId = docValues.get(docId); + BytesRef parentId = docValues.binaryValue(); return parentId.length > 0 ? parentId.utf8ToString() : null; } catch (IOException e) { throw ExceptionsHelper.convertToElastic(e); diff --git a/core/src/main/java/org/elasticsearch/search/fetch/subphase/VersionFetchSubPhase.java b/core/src/main/java/org/elasticsearch/search/fetch/subphase/VersionFetchSubPhase.java index 1ce102e364b4b..5f69230ca42fd 100644 --- a/core/src/main/java/org/elasticsearch/search/fetch/subphase/VersionFetchSubPhase.java +++ b/core/src/main/java/org/elasticsearch/search/fetch/subphase/VersionFetchSubPhase.java @@ -38,8 +38,8 @@ public void hitExecute(SearchContext context, HitContext hitContext) { long version = Versions.NOT_FOUND; try { NumericDocValues versions = hitContext.reader().getNumericDocValues(VersionFieldMapper.NAME); - if (versions != null) { - version = versions.get(hitContext.docId()); + if (versions != null && versions.advanceExact(hitContext.docId())) { + version = versions.longValue(); } } catch (IOException e) { throw new ElasticsearchException("Could not retrieve version", e); From 1816b262d65d533744e3115262d5356400c73d75 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Thu, 6 Apr 2017 10:01:51 +0200 Subject: [PATCH 20/87] Deprecate disable_coord. --- .../index/query/BoolQueryBuilder.java | 40 +++++-------------- .../index/query/CommonTermsQueryBuilder.java | 33 ++++++--------- .../index/query/BoolQueryBuilderTests.java | 8 +--- .../query/CommonTermsQueryBuilderTests.java | 8 +--- .../migration/migrate_6_0/search.asciidoc | 3 ++ docs/reference/query-dsl/bool-query.asciidoc | 5 --- .../query-dsl/common-terms-query.asciidoc | 4 +- 7 files changed, 30 insertions(+), 71 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/index/query/BoolQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/BoolQueryBuilder.java index 5eb8b81009cf0..dcab99ce6a10d 100644 --- a/core/src/main/java/org/elasticsearch/index/query/BoolQueryBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/query/BoolQueryBuilder.java @@ -24,6 +24,7 @@ import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.Query; +import org.elasticsearch.Version; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.io.stream.StreamInput; @@ -48,14 +49,14 @@ public class BoolQueryBuilder extends AbstractQueryBuilder { public static final String NAME = "bool"; public static final boolean ADJUST_PURE_NEGATIVE_DEFAULT = true; - public static final boolean DISABLE_COORD_DEFAULT = false; private static final String MUSTNOT = "mustNot"; private static final String MUST_NOT = "must_not"; private static final String FILTER = "filter"; private static final String SHOULD = "should"; private static final String MUST = "must"; - private static final ParseField DISABLE_COORD_FIELD = new ParseField("disable_coord"); + private static final ParseField DISABLE_COORD_FIELD = new ParseField("disable_coord") + .withAllDeprecated("disable_coord has been removed"); private static final ParseField MINIMUM_SHOULD_MATCH = new ParseField("minimum_should_match"); private static final ParseField ADJUST_PURE_NEGATIVE = new ParseField("adjust_pure_negative"); @@ -67,8 +68,6 @@ public class BoolQueryBuilder extends AbstractQueryBuilder { private final List shouldClauses = new ArrayList<>(); - private boolean disableCoord = DISABLE_COORD_DEFAULT; - private boolean adjustPureNegative = ADJUST_PURE_NEGATIVE_DEFAULT; private String minimumShouldMatch; @@ -89,7 +88,9 @@ public BoolQueryBuilder(StreamInput in) throws IOException { shouldClauses.addAll(readQueries(in)); filterClauses.addAll(readQueries(in)); adjustPureNegative = in.readBoolean(); - disableCoord = in.readBoolean(); + if (in.getVersion().before(Version.V_6_0_0_alpha1_UNRELEASED)) { + in.readBoolean(); // disable_coord + } minimumShouldMatch = in.readOptionalString(); } @@ -100,7 +101,9 @@ protected void doWriteTo(StreamOutput out) throws IOException { writeQueries(out, shouldClauses); writeQueries(out, filterClauses); out.writeBoolean(adjustPureNegative); - out.writeBoolean(disableCoord); + if (out.getVersion().before(Version.V_6_0_0_alpha1_UNRELEASED)) { + out.writeBoolean(true); // disable_coord + } out.writeOptionalString(minimumShouldMatch); } @@ -186,21 +189,6 @@ public List should() { return this.shouldClauses; } - /** - * Disables Similarity#coord(int,int) in scoring. Defaults to false. - */ - public BoolQueryBuilder disableCoord(boolean disableCoord) { - this.disableCoord = disableCoord; - return this; - } - - /** - * @return whether the Similarity#coord(int,int) in scoring are disabled. Defaults to false. - */ - public boolean disableCoord() { - return this.disableCoord; - } - /** * @return the string representation of the minimumShouldMatch settings for this query */ @@ -267,7 +255,6 @@ protected void doXContent(XContentBuilder builder, Params params) throws IOExcep doXArrayContent(FILTER, filterClauses, builder, params); doXArrayContent(MUST_NOT, mustNotClauses, builder, params); doXArrayContent(SHOULD, shouldClauses, builder, params); - builder.field(DISABLE_COORD_FIELD.getPreferredName(), disableCoord); builder.field(ADJUST_PURE_NEGATIVE.getPreferredName(), adjustPureNegative); if (minimumShouldMatch != null) { builder.field(MINIMUM_SHOULD_MATCH.getPreferredName(), minimumShouldMatch); @@ -291,7 +278,6 @@ private static void doXArrayContent(String field, List clauses, XC public static BoolQueryBuilder fromXContent(QueryParseContext parseContext) throws IOException, ParsingException { XContentParser parser = parseContext.parser(); - boolean disableCoord = BoolQueryBuilder.DISABLE_COORD_DEFAULT; boolean adjustPureNegative = BoolQueryBuilder.ADJUST_PURE_NEGATIVE_DEFAULT; float boost = AbstractQueryBuilder.DEFAULT_BOOST; String minimumShouldMatch = null; @@ -349,7 +335,7 @@ public static BoolQueryBuilder fromXContent(QueryParseContext parseContext) thro } } else if (token.isValue()) { if (DISABLE_COORD_FIELD.match(currentFieldName)) { - disableCoord = parser.booleanValue(); + // ignore } else if (MINIMUM_SHOULD_MATCH.match(currentFieldName)) { minimumShouldMatch = parser.textOrNull(); } else if (AbstractQueryBuilder.BOOST_FIELD.match(currentFieldName)) { @@ -377,7 +363,6 @@ public static BoolQueryBuilder fromXContent(QueryParseContext parseContext) thro boolQuery.filter(queryBuilder); } boolQuery.boost(boost); - boolQuery.disableCoord(disableCoord); boolQuery.adjustPureNegative(adjustPureNegative); boolQuery.minimumShouldMatch(minimumShouldMatch); boolQuery.queryName(queryName); @@ -392,7 +377,6 @@ public String getWriteableName() { @Override protected Query doToQuery(QueryShardContext context) throws IOException { BooleanQuery.Builder booleanQueryBuilder = new BooleanQuery.Builder(); - booleanQueryBuilder.setDisableCoord(disableCoord); addBooleanClauses(context, booleanQueryBuilder, mustClauses, BooleanClause.Occur.MUST); addBooleanClauses(context, booleanQueryBuilder, mustNotClauses, BooleanClause.Occur.MUST_NOT); addBooleanClauses(context, booleanQueryBuilder, shouldClauses, BooleanClause.Occur.SHOULD); @@ -432,14 +416,13 @@ private static void addBooleanClauses(QueryShardContext context, BooleanQuery.Bu @Override protected int doHashCode() { - return Objects.hash(adjustPureNegative, disableCoord, + return Objects.hash(adjustPureNegative, minimumShouldMatch, mustClauses, shouldClauses, mustNotClauses, filterClauses); } @Override protected boolean doEquals(BoolQueryBuilder other) { return Objects.equals(adjustPureNegative, other.adjustPureNegative) && - Objects.equals(disableCoord, other.disableCoord) && Objects.equals(minimumShouldMatch, other.minimumShouldMatch) && Objects.equals(mustClauses, other.mustClauses) && Objects.equals(shouldClauses, other.shouldClauses) && @@ -462,7 +445,6 @@ protected QueryBuilder doRewrite(QueryRewriteContext queryRewriteContext) throws if (changed) { newBuilder.adjustPureNegative = adjustPureNegative; - newBuilder.disableCoord = disableCoord; newBuilder.minimumShouldMatch = minimumShouldMatch; newBuilder.boost(boost()); newBuilder.queryName(queryName()); diff --git a/core/src/main/java/org/elasticsearch/index/query/CommonTermsQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/CommonTermsQueryBuilder.java index a91d8fe44cf57..cd2a9c1193940 100644 --- a/core/src/main/java/org/elasticsearch/index/query/CommonTermsQueryBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/query/CommonTermsQueryBuilder.java @@ -29,6 +29,7 @@ import org.apache.lucene.search.Query; import org.apache.lucene.search.similarities.Similarity; import org.apache.lucene.util.BytesRefBuilder; +import org.elasticsearch.Version; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.Strings; @@ -69,7 +70,8 @@ public class CommonTermsQueryBuilder extends AbstractQueryBuilder Date: Thu, 6 Apr 2017 10:16:09 +0200 Subject: [PATCH 21/87] More fixes. --- .../search/ESToParentBlockJoinQuery.java | 4 +- .../search/internal/ContextIndexSearcher.java | 6 +-- .../search/slice/DocValuesSliceQuery.java | 40 +++++++++++++------ .../search/slice/TermsSliceQuery.java | 4 +- 4 files changed, 34 insertions(+), 20 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/index/search/ESToParentBlockJoinQuery.java b/core/src/main/java/org/elasticsearch/index/search/ESToParentBlockJoinQuery.java index 1ee427599cab2..3762b1fffc067 100644 --- a/core/src/main/java/org/elasticsearch/index/search/ESToParentBlockJoinQuery.java +++ b/core/src/main/java/org/elasticsearch/index/search/ESToParentBlockJoinQuery.java @@ -75,8 +75,8 @@ public Query rewrite(IndexReader reader) throws IOException { } @Override - public Weight createWeight(IndexSearcher searcher, boolean needsScores) throws IOException { - return query.createWeight(searcher, needsScores); + public Weight createWeight(IndexSearcher searcher, boolean needsScores, float boost) throws IOException { + return query.createWeight(searcher, needsScores, boost); } @Override diff --git a/core/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java b/core/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java index 8d33140e3eec7..d3b1951846c18 100644 --- a/core/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java +++ b/core/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java @@ -110,7 +110,7 @@ public Weight createNormalizedWeight(Query query, boolean needsScores) throws IO } @Override - public Weight createWeight(Query query, boolean needsScores) throws IOException { + public Weight createWeight(Query query, boolean needsScores, float boost) throws IOException { if (profiler != null) { // createWeight() is called for each query in the tree, so we tell the queryProfiler // each invocation so that it can build an internal representation of the query @@ -119,7 +119,7 @@ public Weight createWeight(Query query, boolean needsScores) throws IOException profile.startTime(QueryTimingType.CREATE_WEIGHT); final Weight weight; try { - weight = super.createWeight(query, needsScores); + weight = super.createWeight(query, needsScores, boost); } finally { profile.stopAndRecordTime(); profiler.pollLastElement(); @@ -127,7 +127,7 @@ public Weight createWeight(Query query, boolean needsScores) throws IOException return new ProfileWeight(query, weight, profile); } else { // needs to be 'super', not 'in' in order to use aggregated DFS - return super.createWeight(query, needsScores); + return super.createWeight(query, needsScores, boost); } } diff --git a/core/src/main/java/org/elasticsearch/search/slice/DocValuesSliceQuery.java b/core/src/main/java/org/elasticsearch/search/slice/DocValuesSliceQuery.java index 95cafacedde99..796bcb2dac864 100644 --- a/core/src/main/java/org/elasticsearch/search/slice/DocValuesSliceQuery.java +++ b/core/src/main/java/org/elasticsearch/search/slice/DocValuesSliceQuery.java @@ -23,10 +23,13 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.DocValues; import org.apache.lucene.index.SortedNumericDocValues; +import org.apache.lucene.search.ConstantScoreScorer; +import org.apache.lucene.search.ConstantScoreWeight; +import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Scorer; +import org.apache.lucene.search.TwoPhaseIterator; import org.apache.lucene.search.Weight; -import org.apache.lucene.search.RandomAccessWeight; -import org.apache.lucene.util.Bits; import java.io.IOException; @@ -42,27 +45,38 @@ public DocValuesSliceQuery(String field, int id, int max) { } @Override - public Weight createWeight(IndexSearcher searcher, boolean needsScores) throws IOException { - return new RandomAccessWeight(this) { + public Weight createWeight(IndexSearcher searcher, boolean needsScores, float boost) throws IOException { + return new ConstantScoreWeight(this, boost) { + @Override - protected Bits getMatchingDocs(final LeafReaderContext context) throws IOException { + public Scorer scorer(LeafReaderContext context) throws IOException { final SortedNumericDocValues values = DocValues.getSortedNumeric(context.reader(), getField()); - return new Bits() { + final DocIdSetIterator approximation = DocIdSetIterator.all(context.reader().maxDoc()); + final TwoPhaseIterator twoPhase = new TwoPhaseIterator(approximation) { + @Override - public boolean get(int doc) { - values.setDocument(doc); - for (int i = 0; i < values.count(); i++) { - return contains(BitMixer.mix(values.valueAt(i))); + public boolean matches() throws IOException { + if (values.advanceExact(approximation.docID())) { + for (int i = 0; i < values.docValueCount(); i++) { + if (contains(BitMixer.mix(values.nextValue()))) { + return true; + } + } + return false; + } else { + return contains(0); } - return contains(0); } @Override - public int length() { - return context.reader().maxDoc(); + public float matchCost() { + // BitMixer.mix seems to be about 10 ops + return 10; } }; + return new ConstantScoreScorer(this, score(), twoPhase); } + }; } } diff --git a/core/src/main/java/org/elasticsearch/search/slice/TermsSliceQuery.java b/core/src/main/java/org/elasticsearch/search/slice/TermsSliceQuery.java index ddc02d32e55a9..bb6908c99fde6 100644 --- a/core/src/main/java/org/elasticsearch/search/slice/TermsSliceQuery.java +++ b/core/src/main/java/org/elasticsearch/search/slice/TermsSliceQuery.java @@ -55,8 +55,8 @@ public TermsSliceQuery(String field, int id, int max) { } @Override - public Weight createWeight(IndexSearcher searcher, boolean needsScores) throws IOException { - return new ConstantScoreWeight(this) { + public Weight createWeight(IndexSearcher searcher, boolean needsScores, float boost) throws IOException { + return new ConstantScoreWeight(this, boost) { @Override public Scorer scorer(LeafReaderContext context) throws IOException { final DocIdSet disi = build(context.reader()); From 2941f219bcb7408e18d19c6805ecb44cf4413f20 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Thu, 6 Apr 2017 10:33:39 +0200 Subject: [PATCH 22/87] Remove ES versions that reference unsupported Lucene versions. --- .../main/java/org/elasticsearch/Version.java | 84 ------------------- 1 file changed, 84 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/Version.java b/core/src/main/java/org/elasticsearch/Version.java index 735043a3a28fd..bf4d9d1f003ca 100644 --- a/core/src/main/java/org/elasticsearch/Version.java +++ b/core/src/main/java/org/elasticsearch/Version.java @@ -35,48 +35,6 @@ public class Version implements Comparable { * values below 25 are for alpha builder (since 5.0), and above 25 and below 50 are beta builds, and below 99 are RC builds, with 99 * indicating a release the (internal) format of the id is there so we can easily do after/before checks on the id */ - public static final int V_2_0_0_ID = 2000099; - public static final Version V_2_0_0 = new Version(V_2_0_0_ID, org.apache.lucene.util.Version.LUCENE_5_2_1); - public static final int V_2_0_1_ID = 2000199; - public static final Version V_2_0_1 = new Version(V_2_0_1_ID, org.apache.lucene.util.Version.LUCENE_5_2_1); - public static final int V_2_0_2_ID = 2000299; - public static final Version V_2_0_2 = new Version(V_2_0_2_ID, org.apache.lucene.util.Version.LUCENE_5_2_1); - public static final int V_2_1_0_ID = 2010099; - public static final Version V_2_1_0 = new Version(V_2_1_0_ID, org.apache.lucene.util.Version.LUCENE_5_3_1); - public static final int V_2_1_1_ID = 2010199; - public static final Version V_2_1_1 = new Version(V_2_1_1_ID, org.apache.lucene.util.Version.LUCENE_5_3_1); - public static final int V_2_1_2_ID = 2010299; - public static final Version V_2_1_2 = new Version(V_2_1_2_ID, org.apache.lucene.util.Version.LUCENE_5_3_1); - public static final int V_2_2_0_ID = 2020099; - public static final Version V_2_2_0 = new Version(V_2_2_0_ID, org.apache.lucene.util.Version.LUCENE_5_4_1); - public static final int V_2_2_1_ID = 2020199; - public static final Version V_2_2_1 = new Version(V_2_2_1_ID, org.apache.lucene.util.Version.LUCENE_5_4_1); - public static final int V_2_2_2_ID = 2020299; - public static final Version V_2_2_2 = new Version(V_2_2_2_ID, org.apache.lucene.util.Version.LUCENE_5_4_1); - public static final int V_2_3_0_ID = 2030099; - public static final Version V_2_3_0 = new Version(V_2_3_0_ID, org.apache.lucene.util.Version.LUCENE_5_5_0); - public static final int V_2_3_1_ID = 2030199; - public static final Version V_2_3_1 = new Version(V_2_3_1_ID, org.apache.lucene.util.Version.LUCENE_5_5_0); - public static final int V_2_3_2_ID = 2030299; - public static final Version V_2_3_2 = new Version(V_2_3_2_ID, org.apache.lucene.util.Version.LUCENE_5_5_0); - public static final int V_2_3_3_ID = 2030399; - public static final Version V_2_3_3 = new Version(V_2_3_3_ID, org.apache.lucene.util.Version.LUCENE_5_5_0); - public static final int V_2_3_4_ID = 2030499; - public static final Version V_2_3_4 = new Version(V_2_3_4_ID, org.apache.lucene.util.Version.LUCENE_5_5_0); - public static final int V_2_3_5_ID = 2030599; - public static final Version V_2_3_5 = new Version(V_2_3_5_ID, org.apache.lucene.util.Version.LUCENE_5_5_0); - public static final int V_2_4_0_ID = 2040099; - public static final Version V_2_4_0 = new Version(V_2_4_0_ID, org.apache.lucene.util.Version.LUCENE_5_5_2); - public static final int V_2_4_1_ID = 2040199; - public static final Version V_2_4_1 = new Version(V_2_4_1_ID, org.apache.lucene.util.Version.LUCENE_5_5_2); - public static final int V_2_4_2_ID = 2040299; - public static final Version V_2_4_2 = new Version(V_2_4_2_ID, org.apache.lucene.util.Version.LUCENE_5_5_2); - public static final int V_2_4_3_ID = 2040399; - public static final Version V_2_4_3 = new Version(V_2_4_3_ID, org.apache.lucene.util.Version.LUCENE_5_5_2); - public static final int V_2_4_4_ID = 2040499; - public static final Version V_2_4_4 = new Version(V_2_4_4_ID, org.apache.lucene.util.Version.LUCENE_5_5_2); - public static final int V_2_4_5_ID = 2040599; - public static final Version V_2_4_5 = new Version(V_2_4_5_ID, org.apache.lucene.util.Version.LUCENE_5_5_2); public static final int V_5_0_0_alpha1_ID = 5000001; public static final Version V_5_0_0_alpha1 = new Version(V_5_0_0_alpha1_ID, org.apache.lucene.util.Version.LUCENE_6_0_0); public static final int V_5_0_0_alpha2_ID = 5000002; @@ -182,48 +140,6 @@ public static Version fromId(int id) { return V_5_0_0_alpha2; case V_5_0_0_alpha1_ID: return V_5_0_0_alpha1; - case V_2_4_5_ID: - return V_2_4_5; - case V_2_4_4_ID: - return V_2_4_4; - case V_2_4_3_ID: - return V_2_4_3; - case V_2_4_2_ID: - return V_2_4_2; - case V_2_4_1_ID: - return V_2_4_1; - case V_2_4_0_ID: - return V_2_4_0; - case V_2_3_5_ID: - return V_2_3_5; - case V_2_3_4_ID: - return V_2_3_4; - case V_2_3_3_ID: - return V_2_3_3; - case V_2_3_2_ID: - return V_2_3_2; - case V_2_3_1_ID: - return V_2_3_1; - case V_2_3_0_ID: - return V_2_3_0; - case V_2_2_2_ID: - return V_2_2_2; - case V_2_2_1_ID: - return V_2_2_1; - case V_2_2_0_ID: - return V_2_2_0; - case V_2_1_2_ID: - return V_2_1_2; - case V_2_1_1_ID: - return V_2_1_1; - case V_2_1_0_ID: - return V_2_1_0; - case V_2_0_2_ID: - return V_2_0_2; - case V_2_0_1_ID: - return V_2_0_1; - case V_2_0_0_ID: - return V_2_0_0; default: return new Version(id, org.apache.lucene.util.Version.LATEST); } From 7e8631959da05ef80263576f42e01efd73364cd5 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Thu, 6 Apr 2017 10:36:16 +0200 Subject: [PATCH 23/87] Revert "Remove ES versions that reference unsupported Lucene versions." This reverts commit 2941f219bcb7408e18d19c6805ecb44cf4413f20. --- .../main/java/org/elasticsearch/Version.java | 84 +++++++++++++++++++ 1 file changed, 84 insertions(+) diff --git a/core/src/main/java/org/elasticsearch/Version.java b/core/src/main/java/org/elasticsearch/Version.java index bf4d9d1f003ca..735043a3a28fd 100644 --- a/core/src/main/java/org/elasticsearch/Version.java +++ b/core/src/main/java/org/elasticsearch/Version.java @@ -35,6 +35,48 @@ public class Version implements Comparable { * values below 25 are for alpha builder (since 5.0), and above 25 and below 50 are beta builds, and below 99 are RC builds, with 99 * indicating a release the (internal) format of the id is there so we can easily do after/before checks on the id */ + public static final int V_2_0_0_ID = 2000099; + public static final Version V_2_0_0 = new Version(V_2_0_0_ID, org.apache.lucene.util.Version.LUCENE_5_2_1); + public static final int V_2_0_1_ID = 2000199; + public static final Version V_2_0_1 = new Version(V_2_0_1_ID, org.apache.lucene.util.Version.LUCENE_5_2_1); + public static final int V_2_0_2_ID = 2000299; + public static final Version V_2_0_2 = new Version(V_2_0_2_ID, org.apache.lucene.util.Version.LUCENE_5_2_1); + public static final int V_2_1_0_ID = 2010099; + public static final Version V_2_1_0 = new Version(V_2_1_0_ID, org.apache.lucene.util.Version.LUCENE_5_3_1); + public static final int V_2_1_1_ID = 2010199; + public static final Version V_2_1_1 = new Version(V_2_1_1_ID, org.apache.lucene.util.Version.LUCENE_5_3_1); + public static final int V_2_1_2_ID = 2010299; + public static final Version V_2_1_2 = new Version(V_2_1_2_ID, org.apache.lucene.util.Version.LUCENE_5_3_1); + public static final int V_2_2_0_ID = 2020099; + public static final Version V_2_2_0 = new Version(V_2_2_0_ID, org.apache.lucene.util.Version.LUCENE_5_4_1); + public static final int V_2_2_1_ID = 2020199; + public static final Version V_2_2_1 = new Version(V_2_2_1_ID, org.apache.lucene.util.Version.LUCENE_5_4_1); + public static final int V_2_2_2_ID = 2020299; + public static final Version V_2_2_2 = new Version(V_2_2_2_ID, org.apache.lucene.util.Version.LUCENE_5_4_1); + public static final int V_2_3_0_ID = 2030099; + public static final Version V_2_3_0 = new Version(V_2_3_0_ID, org.apache.lucene.util.Version.LUCENE_5_5_0); + public static final int V_2_3_1_ID = 2030199; + public static final Version V_2_3_1 = new Version(V_2_3_1_ID, org.apache.lucene.util.Version.LUCENE_5_5_0); + public static final int V_2_3_2_ID = 2030299; + public static final Version V_2_3_2 = new Version(V_2_3_2_ID, org.apache.lucene.util.Version.LUCENE_5_5_0); + public static final int V_2_3_3_ID = 2030399; + public static final Version V_2_3_3 = new Version(V_2_3_3_ID, org.apache.lucene.util.Version.LUCENE_5_5_0); + public static final int V_2_3_4_ID = 2030499; + public static final Version V_2_3_4 = new Version(V_2_3_4_ID, org.apache.lucene.util.Version.LUCENE_5_5_0); + public static final int V_2_3_5_ID = 2030599; + public static final Version V_2_3_5 = new Version(V_2_3_5_ID, org.apache.lucene.util.Version.LUCENE_5_5_0); + public static final int V_2_4_0_ID = 2040099; + public static final Version V_2_4_0 = new Version(V_2_4_0_ID, org.apache.lucene.util.Version.LUCENE_5_5_2); + public static final int V_2_4_1_ID = 2040199; + public static final Version V_2_4_1 = new Version(V_2_4_1_ID, org.apache.lucene.util.Version.LUCENE_5_5_2); + public static final int V_2_4_2_ID = 2040299; + public static final Version V_2_4_2 = new Version(V_2_4_2_ID, org.apache.lucene.util.Version.LUCENE_5_5_2); + public static final int V_2_4_3_ID = 2040399; + public static final Version V_2_4_3 = new Version(V_2_4_3_ID, org.apache.lucene.util.Version.LUCENE_5_5_2); + public static final int V_2_4_4_ID = 2040499; + public static final Version V_2_4_4 = new Version(V_2_4_4_ID, org.apache.lucene.util.Version.LUCENE_5_5_2); + public static final int V_2_4_5_ID = 2040599; + public static final Version V_2_4_5 = new Version(V_2_4_5_ID, org.apache.lucene.util.Version.LUCENE_5_5_2); public static final int V_5_0_0_alpha1_ID = 5000001; public static final Version V_5_0_0_alpha1 = new Version(V_5_0_0_alpha1_ID, org.apache.lucene.util.Version.LUCENE_6_0_0); public static final int V_5_0_0_alpha2_ID = 5000002; @@ -140,6 +182,48 @@ public static Version fromId(int id) { return V_5_0_0_alpha2; case V_5_0_0_alpha1_ID: return V_5_0_0_alpha1; + case V_2_4_5_ID: + return V_2_4_5; + case V_2_4_4_ID: + return V_2_4_4; + case V_2_4_3_ID: + return V_2_4_3; + case V_2_4_2_ID: + return V_2_4_2; + case V_2_4_1_ID: + return V_2_4_1; + case V_2_4_0_ID: + return V_2_4_0; + case V_2_3_5_ID: + return V_2_3_5; + case V_2_3_4_ID: + return V_2_3_4; + case V_2_3_3_ID: + return V_2_3_3; + case V_2_3_2_ID: + return V_2_3_2; + case V_2_3_1_ID: + return V_2_3_1; + case V_2_3_0_ID: + return V_2_3_0; + case V_2_2_2_ID: + return V_2_2_2; + case V_2_2_1_ID: + return V_2_2_1; + case V_2_2_0_ID: + return V_2_2_0; + case V_2_1_2_ID: + return V_2_1_2; + case V_2_1_1_ID: + return V_2_1_1; + case V_2_1_0_ID: + return V_2_1_0; + case V_2_0_2_ID: + return V_2_0_2; + case V_2_0_1_ID: + return V_2_0_1; + case V_2_0_0_ID: + return V_2_0_0; default: return new Version(id, org.apache.lucene.util.Version.LATEST); } From 3138af022294cde97badb9ac52ed38175234d304 Mon Sep 17 00:00:00 2001 From: Colin Goodheart-Smithe Date: Thu, 6 Apr 2017 09:59:10 +0100 Subject: [PATCH 24/87] Removes all remaining compile errors from aggregations package --- .../index/fielddata/FieldData.java | 52 ++++- .../script/LeafSearchScript.java | 2 +- .../scripted/ScriptedMetricAggregator.java | 5 +- .../aggregations/support/MissingValues.java | 195 ++++++++++-------- .../aggregations/support/ValuesSource.java | 77 +++---- .../support/values/ScriptBytesValues.java | 48 +++-- .../support/values/ScriptDoubleValues.java | 57 ++--- .../support/values/ScriptLongValues.java | 55 ++--- .../expression/ExpressionSearchScript.java | 4 +- 9 files changed, 278 insertions(+), 217 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/FieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/FieldData.java index 1d7e10bfcffbe..86e375ad95f50 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/FieldData.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/FieldData.java @@ -114,7 +114,30 @@ public int length() { } /** - * Returns a Bits representing all documents from dv that have a value. + * Returns a {@link Bits} representing all documents from dv + * that have a value. + */ + public static Bits docsWithValue(final SortedSetDocValues dv, final int maxDoc) { + return new Bits() { + @Override + public boolean get(int index) { + try { + return dv.advanceExact(index); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Override + public int length() { + return maxDoc; + } + }; + } + + /** + * Returns a Bits representing all documents from dv that have + * a value. */ public static Bits docsWithValue(final MultiGeoPointValues dv, final int maxDoc) { return new Bits() { @@ -156,8 +179,31 @@ public int length() { } /** - * Given a {@link SortedNumericDoubleValues}, return a {@link SortedNumericDocValues} - * instance that will translate double values to sortable long bits using + * Returns a Bits representing all documents from dv that have + * a value. + */ + public static Bits docsWithValue(final SortedNumericDocValues dv, final int maxDoc) { + return new Bits() { + @Override + public boolean get(int index) { + try { + return dv.advanceExact(index); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Override + public int length() { + return maxDoc; + } + }; + } + + /** + * Given a {@link SortedNumericDoubleValues}, return a + * {@link SortedNumericDocValues} instance that will translate double values + * to sortable long bits using * {@link NumericUtils#doubleToSortableLong(double)}. */ public static SortedNumericDocValues toSortableLongBits(SortedNumericDoubleValues values) { diff --git a/core/src/main/java/org/elasticsearch/script/LeafSearchScript.java b/core/src/main/java/org/elasticsearch/script/LeafSearchScript.java index 0bf9e0d50e072..7fc54801a9560 100644 --- a/core/src/main/java/org/elasticsearch/script/LeafSearchScript.java +++ b/core/src/main/java/org/elasticsearch/script/LeafSearchScript.java @@ -28,7 +28,7 @@ */ public interface LeafSearchScript extends ScorerAware, ExecutableScript { - void setDocument(int doc); + boolean advanceExact(int doc); void setSource(Map source); diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregator.java index 3bfc057682a2a..413e397ca1d34 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregator.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregator.java @@ -67,8 +67,9 @@ public LeafBucketCollector getLeafCollector(LeafReaderContext ctx, @Override public void collect(int doc, long bucket) throws IOException { assert bucket == 0 : bucket; - leafMapScript.setDocument(doc); - leafMapScript.run(); + if (leafMapScript.advanceExact(doc)) { + leafMapScript.run(); + } } }; } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/support/MissingValues.java b/core/src/main/java/org/elasticsearch/search/aggregations/support/MissingValues.java index 4a01b67b78078..a34860ee4496c 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/support/MissingValues.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/support/MissingValues.java @@ -20,11 +20,12 @@ package org.elasticsearch.search.aggregations.support; import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.RandomAccessOrds; import org.apache.lucene.index.SortedNumericDocValues; +import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.geo.GeoPoint; -import org.elasticsearch.index.fielddata.AbstractRandomAccessOrds; +import org.elasticsearch.index.fielddata.AbstractSortedNumericDocValues; +import org.elasticsearch.index.fielddata.AbstractSortedSetDocValues; import org.elasticsearch.index.fielddata.MultiGeoPointValues; import org.elasticsearch.index.fielddata.SortedBinaryDocValues; import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; @@ -56,25 +57,26 @@ static SortedBinaryDocValues replaceMissing(final SortedBinaryDocValues values, private int count; @Override - public BytesRef valueAt(int index) { - if (count > 0) { - return values.valueAt(index); - } else if (index == 0) { - return missing; - } else { - throw new IndexOutOfBoundsException(); - } + public boolean advanceExact(int doc) throws IOException { + values.advanceExact(doc); + count = values.docValueCount(); + // always return true because we want to return a value even if + // the document does not have a value + return true; } @Override - public void setDocument(int docId) { - values.setDocument(docId); - count = values.count(); + public int docValueCount() { + return count == 0 ? 1 : count; } @Override - public int count() { - return count == 0 ? 1 : count; + public BytesRef nextValue() throws IOException { + if (count > 0) { + return values.nextValue(); + } else { + return missing; + } } }; } @@ -109,32 +111,33 @@ public SortedNumericDoubleValues doubleValues(LeafReaderContext context) throws } static SortedNumericDocValues replaceMissing(final SortedNumericDocValues values, final long missing) { - return new SortedNumericDocValues() { + return new AbstractSortedNumericDocValues() { private int count; @Override - public void setDocument(int doc) { - values.setDocument(doc); - count = values.count(); - } - - @Override - public long valueAt(int index) { + public long nextValue() throws IOException { if (count > 0) { - return values.valueAt(index); - } else if (index == 0) { - return missing; + return values.nextValue(); } else { - throw new IndexOutOfBoundsException(); + return missing; } } @Override - public int count() { + public int docValueCount() { return count == 0 ? 1 : count; } + @Override + public boolean advanceExact(int doc) throws IOException { + values.advanceExact(doc); + count = values.docValueCount(); + // always return true because we want to return a value even if + // the document does not have a value + return true; + } + }; } @@ -144,24 +147,25 @@ static SortedNumericDoubleValues replaceMissing(final SortedNumericDoubleValues private int count; @Override - public void setDocument(int doc) { - values.setDocument(doc); - count = values.count(); + public boolean advanceExact(int doc) throws IOException { + values.advanceExact(doc); + count = values.docValueCount(); + // always return true because we want to return a value even if + // the document does not have a value + return true; } @Override - public double valueAt(int index) { + public double nextValue() throws IOException { if (count > 0) { - return values.valueAt(index); - } else if (index == 0) { - return missing; + return values.nextValue(); } else { - throw new IndexOutOfBoundsException(); + return missing; } } @Override - public int count() { + public int docValueCount() { return count == 0 ? 1 : count; } @@ -177,20 +181,22 @@ public SortedBinaryDocValues bytesValues(LeafReaderContext context) throws IOExc } @Override - public RandomAccessOrds ordinalsValues(LeafReaderContext context) { - RandomAccessOrds values = valuesSource.ordinalsValues(context); + public SortedSetDocValues ordinalsValues(LeafReaderContext context) throws IOException { + SortedSetDocValues values = valuesSource.ordinalsValues(context); return replaceMissing(values, missing); } @Override - public RandomAccessOrds globalOrdinalsValues(LeafReaderContext context) { - RandomAccessOrds values = valuesSource.globalOrdinalsValues(context); + public SortedSetDocValues globalOrdinalsValues(LeafReaderContext context) + throws IOException { + SortedSetDocValues values = valuesSource.globalOrdinalsValues(context); return replaceMissing(values, missing); } }; } - static RandomAccessOrds replaceMissing(final RandomAccessOrds values, final BytesRef missing) { + static SortedSetDocValues replaceMissing(final SortedSetDocValues values, + final BytesRef missing) throws IOException { final long missingOrd = values.lookupTerm(missing); if (missingOrd >= 0) { // The value already exists @@ -201,19 +207,15 @@ static RandomAccessOrds replaceMissing(final RandomAccessOrds values, final Byte } } - static RandomAccessOrds replaceMissingOrd(final RandomAccessOrds values, final long missingOrd) { - return new AbstractRandomAccessOrds() { - - private int cardinality = 0; + static SortedSetDocValues replaceMissingOrd(final SortedSetDocValues values, + final long missingOrd) { + return new AbstractSortedSetDocValues() { - @Override - public void doSetDocument(int docID) { - values.setDocument(docID); - cardinality = values.cardinality(); - } + private boolean hasOrds; + private long nextMissingOrd; @Override - public BytesRef lookupOrd(long ord) { + public BytesRef lookupOrd(long ord) throws IOException { return values.lookupOrd(ord); } @@ -223,36 +225,39 @@ public long getValueCount() { } @Override - public long ordAt(int index) { - if (cardinality > 0) { - return values.ordAt(index); - } else if (index == 0) { - return missingOrd; + public long nextOrd() throws IOException { + if (hasOrds) { + return values.nextOrd(); } else { - throw new IndexOutOfBoundsException(); + // we want to return the next missing ord but set this to + // NO_MORE_ORDS so on the next call we indicate there are no + // more values + long ordToReturn = nextMissingOrd; + nextMissingOrd = SortedSetDocValues.NO_MORE_ORDS; + return ordToReturn; } } @Override - public int cardinality() { - return cardinality == 0 ? 1 : cardinality; + public boolean advanceExact(int doc) throws IOException { + hasOrds = values.advanceExact(doc); + nextMissingOrd = SortedSetDocValues.NO_MORE_ORDS; + // always return true because we want to return a value even if + // the document does not have a value + return true; } }; } - static RandomAccessOrds insertOrd(final RandomAccessOrds values, final long insertedOrd, final BytesRef missingValue) { - return new AbstractRandomAccessOrds() { + static SortedSetDocValues insertOrd(final SortedSetDocValues values, final long insertedOrd, + final BytesRef missingValue) { + return new AbstractSortedSetDocValues() { - private int cardinality = 0; + private boolean hasOrds; + private long nextMissingOrd; @Override - public void doSetDocument(int docID) { - values.setDocument(docID); - cardinality = values.cardinality(); - } - - @Override - public BytesRef lookupOrd(long ord) { + public BytesRef lookupOrd(long ord) throws IOException { if (ord < insertedOrd) { return values.lookupOrd(ord); } else if (ord > insertedOrd) { @@ -268,24 +273,31 @@ public long getValueCount() { } @Override - public long ordAt(int index) { - if (cardinality > 0) { - final long ord = values.ordAt(index); + public long nextOrd() throws IOException { + if (hasOrds) { + final long ord = values.nextOrd(); if (ord < insertedOrd) { return ord; } else { return ord + 1; } - } else if (index == 0) { - return insertedOrd; } else { - throw new IndexOutOfBoundsException(); + // we want to return the next missing ord but set this to + // NO_MORE_ORDS so on the next call we indicate there are no + // more values + long ordToReturn = nextMissingOrd; + nextMissingOrd = SortedSetDocValues.NO_MORE_ORDS; + return ordToReturn; } } @Override - public int cardinality() { - return cardinality == 0 ? 1 : cardinality; + public boolean advanceExact(int doc) throws IOException { + hasOrds = values.advanceExact(doc); + nextMissingOrd = SortedSetDocValues.NO_MORE_ORDS; + // always return true because we want to return a value even if + // the document does not have a value + return true; } }; } @@ -312,25 +324,26 @@ static MultiGeoPointValues replaceMissing(final MultiGeoPointValues values, fina private int count; @Override - public GeoPoint valueAt(int index) { - if (count > 0) { - return values.valueAt(index); - } else if (index == 0) { - return missing; - } else { - throw new IndexOutOfBoundsException(); - } + public boolean advanceExact(int doc) throws IOException { + values.advanceExact(doc); + count = values.docValueCount(); + // always return true because we want to return a value even if + // the document does not have a value + return true; } @Override - public void setDocument(int docId) { - values.setDocument(docId); - count = values.count(); + public int docValueCount() { + return count == 0 ? 1 : count; } @Override - public int count() { - return count == 0 ? 1 : count; + public GeoPoint nextValue() { + if (count > 0) { + return values.nextValue(); + } else { + return missing; + } } }; } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSource.java b/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSource.java index 672447720ed9f..7f84695903101 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSource.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSource.java @@ -72,11 +72,8 @@ public abstract static class Bytes extends ValuesSource { @Override public Bits docsWithValue(LeafReaderContext context) throws IOException { final SortedBinaryDocValues bytes = bytesValues(context); - if (org.elasticsearch.index.fielddata.FieldData.unwrapSingleton(bytes) != null) { - return org.elasticsearch.index.fielddata.FieldData.unwrapSingletonBits(bytes); - } else { - return org.elasticsearch.index.fielddata.FieldData.docsWithValue(bytes, context.reader().maxDoc()); - } + return org.elasticsearch.index.fielddata.FieldData.docsWithValue(bytes, + context.reader().maxDoc()); } public abstract static class WithOrdinals extends Bytes { @@ -101,20 +98,19 @@ public SortedBinaryDocValues bytesValues(LeafReaderContext context) throws IOExc }; @Override - public Bits docsWithValue(LeafReaderContext context) { + public Bits docsWithValue(LeafReaderContext context) throws IOException { final SortedSetDocValues ordinals = ordinalsValues(context); - if (DocValues.unwrapSingleton(ordinals) != null) { - return DocValues.docsWithValue(DocValues.unwrapSingleton(ordinals), context.reader().maxDoc()); - } else { - return DocValues.docsWithValue(ordinals, context.reader().maxDoc()); - } + return org.elasticsearch.index.fielddata.FieldData.docsWithValue(ordinals, + context.reader().maxDoc()); } - public abstract SortedSetDocValues ordinalsValues(LeafReaderContext context); + public abstract SortedSetDocValues ordinalsValues(LeafReaderContext context) + throws IOException; - public abstract SortedSetDocValues globalOrdinalsValues(LeafReaderContext context); + public abstract SortedSetDocValues globalOrdinalsValues(LeafReaderContext context) + throws IOException; - public long globalMaxOrd(IndexSearcher indexSearcher) { + public long globalMaxOrd(IndexSearcher indexSearcher) throws IOException { IndexReader indexReader = indexSearcher.getIndexReader(); if (indexReader.leaves().isEmpty()) { return 0; @@ -263,18 +259,12 @@ public SortedBinaryDocValues bytesValues(LeafReaderContext context) throws IOExc public Bits docsWithValue(LeafReaderContext context) throws IOException { if (isFloatingPoint()) { final SortedNumericDoubleValues values = doubleValues(context); - if (org.elasticsearch.index.fielddata.FieldData.unwrapSingleton(values) != null) { - return org.elasticsearch.index.fielddata.FieldData.unwrapSingletonBits(values); - } else { - return org.elasticsearch.index.fielddata.FieldData.docsWithValue(values, context.reader().maxDoc()); - } + return org.elasticsearch.index.fielddata.FieldData.docsWithValue(values, + context.reader().maxDoc()); } else { final SortedNumericDocValues values = longValues(context); - if (DocValues.unwrapSingleton(values) != null) { - return DocValues.unwrapSingletonBits(values); - } else { - return DocValues.docsWithValue(values, context.reader().maxDoc()); - } + return org.elasticsearch.index.fielddata.FieldData.docsWithValue(values, + context.reader().maxDoc()); } } @@ -332,16 +322,16 @@ public void setScorer(Scorer scorer) { public boolean advanceExact(int target) throws IOException { if (longValues.advanceExact(target)) { resize(longValues.docValueCount()); - script.setDocument(target); - for (int i = 0; i < docValueCount(); ++i) { - script.setNextAggregationValue(longValues.nextValue()); - values[i] = script.runAsLong(); + if (script.advanceExact(target)) { + for (int i = 0; i < docValueCount(); ++i) { + script.setNextAggregationValue(longValues.nextValue()); + values[i] = script.runAsLong(); + } + sort(); + return true; } - sort(); - return true; - } else { - return false; } + return false; } } @@ -364,16 +354,16 @@ public void setScorer(Scorer scorer) { public boolean advanceExact(int target) throws IOException { if (doubleValues.advanceExact(target)) { resize(doubleValues.docValueCount()); - script.setDocument(target); - for (int i = 0; i < docValueCount(); ++i) { - script.setNextAggregationValue(doubleValues.nextValue()); - values[i] = script.runAsLong(); + if (script.advanceExact(target)) { + for (int i = 0; i < docValueCount(); ++i) { + script.setNextAggregationValue(doubleValues.nextValue()); + values[i] = script.runAsLong(); + } + sort(); + return true; } - sort(); - return true; - } else { - return false; } + return false; } } } @@ -520,11 +510,8 @@ public SortedBinaryDocValues bytesValues(LeafReaderContext context) throws IOExc @Override public Bits docsWithValue(LeafReaderContext context) { final MultiGeoPointValues geoPoints = geoPointValues(context); - if (org.elasticsearch.index.fielddata.FieldData.unwrapSingleton(geoPoints) != null) { - return org.elasticsearch.index.fielddata.FieldData.unwrapSingletonBits(geoPoints); - } else { - return org.elasticsearch.index.fielddata.FieldData.docsWithValue(geoPoints, context.reader().maxDoc()); - } + return org.elasticsearch.index.fielddata.FieldData.docsWithValue(geoPoints, + context.reader().maxDoc()); } public abstract MultiGeoPointValues geoPointValues(LeafReaderContext context); diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/support/values/ScriptBytesValues.java b/core/src/main/java/org/elasticsearch/search/aggregations/support/values/ScriptBytesValues.java index ce36fec941671..b1c0fc9676ad5 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/support/values/ScriptBytesValues.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/support/values/ScriptBytesValues.java @@ -24,6 +24,7 @@ import org.elasticsearch.index.fielddata.SortingBinaryDocValues; import org.elasticsearch.script.LeafSearchScript; +import java.io.IOException; import java.lang.reflect.Array; import java.util.Collection; @@ -48,31 +49,34 @@ private void set(int i, Object o) { } @Override - public void setDocument(int docId) { - script.setDocument(docId); - final Object value = script.run(); - - if (value == null) { - count = 0; - } else if (value.getClass().isArray()) { - count = Array.getLength(value); - grow(); - for (int i = 0; i < count; ++i) { - set(i, Array.get(value, i)); - } - } else if (value instanceof Collection) { - final Collection coll = (Collection) value; - count = coll.size(); - grow(); - int i = 0; - for (Object v : coll) { - set(i++, v); + public boolean advanceExact(int doc) throws IOException { + if (script.advanceExact(doc)) { + final Object value = script.run(); + if (value == null) { + return false; + } else if (value.getClass().isArray()) { + count = Array.getLength(value); + grow(); + for (int i = 0; i < count; ++i) { + set(i, Array.get(value, i)); + } + } else if (value instanceof Collection) { + final Collection coll = (Collection) value; + count = coll.size(); + grow(); + int i = 0; + for (Object v : coll) { + set(i++, v); + } + } else { + count = 1; + set(0, value); } + sort(); + return true; } else { - count = 1; - set(0, value); + return false; } - sort(); } @Override diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/support/values/ScriptDoubleValues.java b/core/src/main/java/org/elasticsearch/search/aggregations/support/values/ScriptDoubleValues.java index 4f91f59ebe7bf..c52af07dd2746 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/support/values/ScriptDoubleValues.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/support/values/ScriptDoubleValues.java @@ -25,6 +25,7 @@ import org.elasticsearch.search.aggregations.AggregationExecutionException; import org.joda.time.ReadableInstant; +import java.io.IOException; import java.lang.reflect.Array; import java.util.Collection; @@ -41,36 +42,40 @@ public ScriptDoubleValues(LeafSearchScript script) { } @Override - public void setDocument(int docId) { - script.setDocument(docId); - final Object value = script.run(); + public boolean advanceExact(int target) throws IOException { + if (script.advanceExact(target)) { + final Object value = script.run(); - if (value == null) { - resize(0); - } else if (value instanceof Number) { - resize(1); - values[0] = ((Number) value).doubleValue(); - } else if (value instanceof ReadableInstant) { - resize(1); - values[0] = ((ReadableInstant) value).getMillis(); - } else if (value.getClass().isArray()) { - resize(Array.getLength(value)); - for (int i = 0; i < count(); ++i) { - values[i] = toDoubleValue(Array.get(value, i)); + if (value == null) { + return false; + } else if (value instanceof Number) { + resize(1); + values[0] = ((Number) value).doubleValue(); + } else if (value instanceof ReadableInstant) { + resize(1); + values[0] = ((ReadableInstant) value).getMillis(); + } else if (value.getClass().isArray()) { + resize(Array.getLength(value)); + for (int i = 0; i < docValueCount(); ++i) { + values[i] = toDoubleValue(Array.get(value, i)); + } + } else if (value instanceof Collection) { + resize(((Collection) value).size()); + int i = 0; + for (Object v : (Collection) value) { + values[i++] = toDoubleValue(v); + } + assert i == docValueCount(); + } else { + resize(1); + values[0] = toDoubleValue(value); } - } else if (value instanceof Collection) { - resize(((Collection) value).size()); - int i = 0; - for (Object v : (Collection) value) { - values[i++] = toDoubleValue(v); - } - assert i == count(); + + sort(); + return true; } else { - resize(1); - values[0] = toDoubleValue(value); + return false; } - - sort(); } private static double toDoubleValue(Object o) { diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/support/values/ScriptLongValues.java b/core/src/main/java/org/elasticsearch/search/aggregations/support/values/ScriptLongValues.java index cd515401c5263..815bf85f92e4f 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/support/values/ScriptLongValues.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/support/values/ScriptLongValues.java @@ -21,11 +21,12 @@ import org.apache.lucene.search.Scorer; import org.apache.lucene.util.LongValues; import org.elasticsearch.common.lucene.ScorerAware; -import org.elasticsearch.index.fielddata.SortingNumericDocValues; +import org.elasticsearch.index.fielddata.AbstractSortingNumericDocValues; import org.elasticsearch.script.LeafSearchScript; import org.elasticsearch.search.aggregations.AggregationExecutionException; import org.joda.time.ReadableInstant; +import java.io.IOException; import java.lang.reflect.Array; import java.util.Collection; import java.util.Iterator; @@ -33,7 +34,7 @@ /** * {@link LongValues} implementation which is based on a script */ -public class ScriptLongValues extends SortingNumericDocValues implements ScorerAware { +public class ScriptLongValues extends AbstractSortingNumericDocValues implements ScorerAware { final LeafSearchScript script; @@ -43,36 +44,40 @@ public ScriptLongValues(LeafSearchScript script) { } @Override - public void setDocument(int docId) { - script.setDocument(docId); - final Object value = script.run(); + public boolean advanceExact(int target) throws IOException { + if (script.advanceExact(target)) { + final Object value = script.run(); - if (value == null) { - resize(0); - } + if (value == null) { + return false; + } - else if (value.getClass().isArray()) { - resize(Array.getLength(value)); - for (int i = 0; i < count(); ++i) { - values[i] = toLongValue(Array.get(value, i)); + else if (value.getClass().isArray()) { + resize(Array.getLength(value)); + for (int i = 0; i < docValueCount(); ++i) { + values[i] = toLongValue(Array.get(value, i)); + } } - } - else if (value instanceof Collection) { - resize(((Collection) value).size()); - int i = 0; - for (Iterator it = ((Collection) value).iterator(); it.hasNext(); ++i) { - values[i] = toLongValue(it.next()); + else if (value instanceof Collection) { + resize(((Collection) value).size()); + int i = 0; + for (Iterator it = ((Collection) value).iterator(); it.hasNext(); ++i) { + values[i] = toLongValue(it.next()); + } + assert i == docValueCount(); } - assert i == count(); - } - else { - resize(1); - values[0] = toLongValue(value); - } + else { + resize(1); + values[0] = toLongValue(value); + } - sort(); + sort(); + return true; + } else { + return false; + } } private static long toLongValue(Object o) { diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionSearchScript.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionSearchScript.java index 11a17d38c5993..2766bb68e2825 100644 --- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionSearchScript.java +++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionSearchScript.java @@ -95,10 +95,10 @@ public boolean advanceExact(int doc) throws IOException { public double runAsDouble() { return evaluate(); } @Override - public void setDocument(int d) { + public boolean advanceExact(int d) { docid = d; try { - values.advanceExact(d); + return values.advanceExact(d); } catch (IOException e) { throw new IllegalStateException("Can't advance to doc using " + compiledScript, e); } From 4bad26288980135c6a383c25836f2d32b14a0be5 Mon Sep 17 00:00:00 2001 From: Colin Goodheart-Smithe Date: Thu, 6 Apr 2017 10:08:47 +0100 Subject: [PATCH 25/87] Reverts LeafSearchScript to use setDocument again This is until we can work out what to do with LeafSearchLookup which seeks to the document on multiple things each of which may or may not have values for the document but also may or may not be used by the running script --- .../script/LeafSearchScript.java | 2 +- .../scripted/ScriptedMetricAggregator.java | 5 +- .../aggregations/support/ValuesSource.java | 26 ++++----- .../support/values/ScriptBytesValues.java | 45 +++++++-------- .../support/values/ScriptDoubleValues.java | 55 +++++++++---------- .../support/values/ScriptLongValues.java | 49 ++++++++--------- .../expression/ExpressionSearchScript.java | 4 +- 7 files changed, 87 insertions(+), 99 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/script/LeafSearchScript.java b/core/src/main/java/org/elasticsearch/script/LeafSearchScript.java index 7fc54801a9560..0bf9e0d50e072 100644 --- a/core/src/main/java/org/elasticsearch/script/LeafSearchScript.java +++ b/core/src/main/java/org/elasticsearch/script/LeafSearchScript.java @@ -28,7 +28,7 @@ */ public interface LeafSearchScript extends ScorerAware, ExecutableScript { - boolean advanceExact(int doc); + void setDocument(int doc); void setSource(Map source); diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregator.java index 413e397ca1d34..3bfc057682a2a 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregator.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregator.java @@ -67,9 +67,8 @@ public LeafBucketCollector getLeafCollector(LeafReaderContext ctx, @Override public void collect(int doc, long bucket) throws IOException { assert bucket == 0 : bucket; - if (leafMapScript.advanceExact(doc)) { - leafMapScript.run(); - } + leafMapScript.setDocument(doc); + leafMapScript.run(); } }; } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSource.java b/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSource.java index 7f84695903101..7fcc7595ea62c 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSource.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSource.java @@ -322,14 +322,13 @@ public void setScorer(Scorer scorer) { public boolean advanceExact(int target) throws IOException { if (longValues.advanceExact(target)) { resize(longValues.docValueCount()); - if (script.advanceExact(target)) { - for (int i = 0; i < docValueCount(); ++i) { - script.setNextAggregationValue(longValues.nextValue()); - values[i] = script.runAsLong(); - } - sort(); - return true; + script.setDocument(target); + for (int i = 0; i < docValueCount(); ++i) { + script.setNextAggregationValue(longValues.nextValue()); + values[i] = script.runAsLong(); } + sort(); + return true; } return false; } @@ -354,14 +353,13 @@ public void setScorer(Scorer scorer) { public boolean advanceExact(int target) throws IOException { if (doubleValues.advanceExact(target)) { resize(doubleValues.docValueCount()); - if (script.advanceExact(target)) { - for (int i = 0; i < docValueCount(); ++i) { - script.setNextAggregationValue(doubleValues.nextValue()); - values[i] = script.runAsLong(); - } - sort(); - return true; + script.setDocument(target); + for (int i = 0; i < docValueCount(); ++i) { + script.setNextAggregationValue(doubleValues.nextValue()); + values[i] = script.runAsLong(); } + sort(); + return true; } return false; } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/support/values/ScriptBytesValues.java b/core/src/main/java/org/elasticsearch/search/aggregations/support/values/ScriptBytesValues.java index b1c0fc9676ad5..9b98097923dfe 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/support/values/ScriptBytesValues.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/support/values/ScriptBytesValues.java @@ -50,33 +50,30 @@ private void set(int i, Object o) { @Override public boolean advanceExact(int doc) throws IOException { - if (script.advanceExact(doc)) { - final Object value = script.run(); - if (value == null) { - return false; - } else if (value.getClass().isArray()) { - count = Array.getLength(value); - grow(); - for (int i = 0; i < count; ++i) { - set(i, Array.get(value, i)); - } - } else if (value instanceof Collection) { - final Collection coll = (Collection) value; - count = coll.size(); - grow(); - int i = 0; - for (Object v : coll) { - set(i++, v); - } - } else { - count = 1; - set(0, value); + script.setDocument(doc); + final Object value = script.run(); + if (value == null) { + return false; + } else if (value.getClass().isArray()) { + count = Array.getLength(value); + grow(); + for (int i = 0; i < count; ++i) { + set(i, Array.get(value, i)); + } + } else if (value instanceof Collection) { + final Collection coll = (Collection) value; + count = coll.size(); + grow(); + int i = 0; + for (Object v : coll) { + set(i++, v); } - sort(); - return true; } else { - return false; + count = 1; + set(0, value); } + sort(); + return true; } @Override diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/support/values/ScriptDoubleValues.java b/core/src/main/java/org/elasticsearch/search/aggregations/support/values/ScriptDoubleValues.java index c52af07dd2746..673e09a4f430c 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/support/values/ScriptDoubleValues.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/support/values/ScriptDoubleValues.java @@ -43,39 +43,36 @@ public ScriptDoubleValues(LeafSearchScript script) { @Override public boolean advanceExact(int target) throws IOException { - if (script.advanceExact(target)) { - final Object value = script.run(); + script.setDocument(target); + final Object value = script.run(); - if (value == null) { - return false; - } else if (value instanceof Number) { - resize(1); - values[0] = ((Number) value).doubleValue(); - } else if (value instanceof ReadableInstant) { - resize(1); - values[0] = ((ReadableInstant) value).getMillis(); - } else if (value.getClass().isArray()) { - resize(Array.getLength(value)); - for (int i = 0; i < docValueCount(); ++i) { - values[i] = toDoubleValue(Array.get(value, i)); - } - } else if (value instanceof Collection) { - resize(((Collection) value).size()); - int i = 0; - for (Object v : (Collection) value) { - values[i++] = toDoubleValue(v); - } - assert i == docValueCount(); - } else { - resize(1); - values[0] = toDoubleValue(value); + if (value == null) { + return false; + } else if (value instanceof Number) { + resize(1); + values[0] = ((Number) value).doubleValue(); + } else if (value instanceof ReadableInstant) { + resize(1); + values[0] = ((ReadableInstant) value).getMillis(); + } else if (value.getClass().isArray()) { + resize(Array.getLength(value)); + for (int i = 0; i < docValueCount(); ++i) { + values[i] = toDoubleValue(Array.get(value, i)); } - - sort(); - return true; + } else if (value instanceof Collection) { + resize(((Collection) value).size()); + int i = 0; + for (Object v : (Collection) value) { + values[i++] = toDoubleValue(v); + } + assert i == docValueCount(); } else { - return false; + resize(1); + values[0] = toDoubleValue(value); } + + sort(); + return true; } private static double toDoubleValue(Object o) { diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/support/values/ScriptLongValues.java b/core/src/main/java/org/elasticsearch/search/aggregations/support/values/ScriptLongValues.java index 815bf85f92e4f..042d987b6e433 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/support/values/ScriptLongValues.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/support/values/ScriptLongValues.java @@ -45,39 +45,36 @@ public ScriptLongValues(LeafSearchScript script) { @Override public boolean advanceExact(int target) throws IOException { - if (script.advanceExact(target)) { - final Object value = script.run(); + script.setDocument(target); + final Object value = script.run(); - if (value == null) { - return false; - } - - else if (value.getClass().isArray()) { - resize(Array.getLength(value)); - for (int i = 0; i < docValueCount(); ++i) { - values[i] = toLongValue(Array.get(value, i)); - } - } + if (value == null) { + return false; + } - else if (value instanceof Collection) { - resize(((Collection) value).size()); - int i = 0; - for (Iterator it = ((Collection) value).iterator(); it.hasNext(); ++i) { - values[i] = toLongValue(it.next()); - } - assert i == docValueCount(); + else if (value.getClass().isArray()) { + resize(Array.getLength(value)); + for (int i = 0; i < docValueCount(); ++i) { + values[i] = toLongValue(Array.get(value, i)); } + } - else { - resize(1); - values[0] = toLongValue(value); + else if (value instanceof Collection) { + resize(((Collection) value).size()); + int i = 0; + for (Iterator it = ((Collection) value).iterator(); it.hasNext(); ++i) { + values[i] = toLongValue(it.next()); } + assert i == docValueCount(); + } - sort(); - return true; - } else { - return false; + else { + resize(1); + values[0] = toLongValue(value); } + + sort(); + return true; } private static long toLongValue(Object o) { diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionSearchScript.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionSearchScript.java index 2766bb68e2825..11a17d38c5993 100644 --- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionSearchScript.java +++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionSearchScript.java @@ -95,10 +95,10 @@ public boolean advanceExact(int doc) throws IOException { public double runAsDouble() { return evaluate(); } @Override - public boolean advanceExact(int d) { + public void setDocument(int d) { docid = d; try { - return values.advanceExact(d); + values.advanceExact(d); } catch (IOException e) { throw new IllegalStateException("Can't advance to doc using " + compiledScript, e); } From fc0144598926d1b14df1e8fd50fd99b1218ca776 Mon Sep 17 00:00:00 2001 From: Colin Goodheart-Smithe Date: Thu, 6 Apr 2017 10:20:00 +0100 Subject: [PATCH 26/87] Fixes matrix agg compile errors --- .../matrix/stats/MatrixStatsAggregator.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsAggregator.java b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsAggregator.java index 4665e74daa72a..df693fc25140e 100644 --- a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsAggregator.java +++ b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsAggregator.java @@ -100,16 +100,16 @@ public void collect(int doc, long bucket) throws IOException { /** * return a map of field names and data */ - private boolean includeDocument(int doc) { + private boolean includeDocument(int doc) throws IOException { // loop over fields for (int i = 0; i < fieldVals.length; ++i) { final NumericDoubleValues doubleValues = values[i]; - final double value = doubleValues.get(doc); - // skip if value is missing - if (value == Double.NEGATIVE_INFINITY) { + if (doubleValues.advanceExact(doc)) { + final double value = doubleValues.doubleValue(); + fieldVals[i] = value; + } else { return false; } - fieldVals[i] = value; } return true; } From 11b7a8a7ad630e5f66e41bf6938b6efa09b80fc9 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Thu, 6 Apr 2017 12:21:31 +0200 Subject: [PATCH 27/87] Fix more compile errors. --- .../PerFieldMappingPostingFormatCodec.java | 4 +- .../elasticsearch/search/MultiValueMode.java | 4 +- .../lucene/queries/BlendedTermQueryTests.java | 12 +- .../common/lucene/LuceneTests.java | 3 +- .../common/lucene/ShardCoreKeyMapTests.java | 4 +- .../lucene/index/ESDirectoryReaderTests.java | 3 +- .../deps/lucene/SimpleLuceneTests.java | 114 +----- .../elasticsearch/index/IndexModuleTests.java | 4 +- .../elasticsearch/index/codec/CodecTests.java | 10 +- .../index/engine/InternalEngineTests.java | 5 +- .../index/mapper/BooleanFieldMapperTests.java | 6 +- .../index/mapper/FakeStringFieldMapper.java | 5 - .../mapper/ScaledFloatFieldTypeTests.java | 14 +- .../query/MultiMatchQueryBuilderTests.java | 2 - .../query/QueryStringQueryBuilderTests.java | 3 - .../index/query/SimpleQueryParserTests.java | 1 - .../functionscore/FunctionScoreTests.java | 20 +- .../query/plugin/DummyQueryParserPlugin.java | 4 +- .../index/search/MultiMatchQueryTests.java | 10 +- .../shard/IndexSearcherWrapperTests.java | 97 +---- .../elasticsearch/index/store/StoreTests.java | 2 +- .../indices/IndicesQueryCacheTests.java | 4 +- .../RandomExceptionCircuitBreakerIT.java | 6 +- .../search/MultiValueModeTests.java | 343 +++++++++++------- .../aggregations/AggregatorTestCase.java | 2 +- .../range/BinaryRangeAggregatorTests.java | 19 +- .../support/ScriptValuesTests.java | 16 +- .../support/ValuesSourceConfigTests.java | 63 ++-- .../basic/SearchWithRandomExceptionsIT.java | 5 + .../subphase/NestedChildrenFilterTests.java | 4 +- .../search/functionscore/QueryRescorerIT.java | 4 +- 31 files changed, 336 insertions(+), 457 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/index/codec/PerFieldMappingPostingFormatCodec.java b/core/src/main/java/org/elasticsearch/index/codec/PerFieldMappingPostingFormatCodec.java index 490c837e85a82..bf1e48e7a6b27 100644 --- a/core/src/main/java/org/elasticsearch/index/codec/PerFieldMappingPostingFormatCodec.java +++ b/core/src/main/java/org/elasticsearch/index/codec/PerFieldMappingPostingFormatCodec.java @@ -23,7 +23,7 @@ import org.apache.lucene.codecs.Codec; import org.apache.lucene.codecs.PostingsFormat; import org.apache.lucene.codecs.lucene50.Lucene50StoredFieldsFormat; -import org.apache.lucene.codecs.lucene62.Lucene62Codec; +import org.apache.lucene.codecs.lucene70.Lucene70Codec; import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.index.mapper.CompletionFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; @@ -38,7 +38,7 @@ * configured for a specific field the default postings format is used. */ // LUCENE UPGRADE: make sure to move to a new codec depending on the lucene version -public class PerFieldMappingPostingFormatCodec extends Lucene62Codec { +public class PerFieldMappingPostingFormatCodec extends Lucene70Codec { private final Logger logger; private final MapperService mapperService; diff --git a/core/src/main/java/org/elasticsearch/search/MultiValueMode.java b/core/src/main/java/org/elasticsearch/search/MultiValueMode.java index 4fab51dcb81ad..c75cbb3c10a4e 100644 --- a/core/src/main/java/org/elasticsearch/search/MultiValueMode.java +++ b/core/src/main/java/org/elasticsearch/search/MultiValueMode.java @@ -196,7 +196,7 @@ protected double pick(UnsortedNumericDoubleValues values) throws IOException { @Override protected long pick(SortedNumericDocValues values) throws IOException { int count = values.docValueCount(); - for (int i = 0; i < count / 2; ++i) { + for (int i = 0; i < (count - 1) / 2; ++i) { values.nextValue(); } if (count % 2 == 0) { @@ -209,7 +209,7 @@ protected long pick(SortedNumericDocValues values) throws IOException { @Override protected double pick(SortedNumericDoubleValues values) throws IOException { int count = values.docValueCount(); - for (int i = 0; i < count / 2; ++i) { + for (int i = 0; i < (count - 1) / 2; ++i) { values.nextValue(); } if (count % 2 == 0) { diff --git a/core/src/test/java/org/apache/lucene/queries/BlendedTermQueryTests.java b/core/src/test/java/org/apache/lucene/queries/BlendedTermQueryTests.java index 3c727ddf6e326..b047716956728 100644 --- a/core/src/test/java/org/apache/lucene/queries/BlendedTermQueryTests.java +++ b/core/src/test/java/org/apache/lucene/queries/BlendedTermQueryTests.java @@ -85,7 +85,7 @@ public void testBooleanQuery() throws IOException { { Term[] terms = new Term[]{new Term("firstname", "simon"), new Term("surname", "simon")}; - BlendedTermQuery query = BlendedTermQuery.booleanBlendedQuery(terms, true); + BlendedTermQuery query = BlendedTermQuery.booleanBlendedQuery(terms); TopDocs search = searcher.search(query, 3); ScoreDoc[] scoreDocs = search.scoreDocs; assertEquals(3, scoreDocs.length); @@ -93,7 +93,6 @@ public void testBooleanQuery() throws IOException { } { BooleanQuery.Builder query = new BooleanQuery.Builder(); - query.setDisableCoord(true); query.add(new TermQuery(new Term("firstname", "simon")), BooleanClause.Occur.SHOULD); query.add(new TermQuery(new Term("surname", "simon")), BooleanClause.Occur.SHOULD); TopDocs search = searcher.search(query.build(), 1); @@ -146,7 +145,6 @@ public void testDismaxQuery() throws IOException { { String[] fields = new String[]{"username", "song"}; BooleanQuery.Builder query = new BooleanQuery.Builder(); - query.setDisableCoord(true); query.add(BlendedTermQuery.dismaxBlendedQuery(toTerms(fields, "foo"), 0.1f), BooleanClause.Occur.SHOULD); query.add(BlendedTermQuery.dismaxBlendedQuery(toTerms(fields, "fighters"), 0.1f), BooleanClause.Occur.SHOULD); query.add(BlendedTermQuery.dismaxBlendedQuery(toTerms(fields, "generator"), 0.1f), BooleanClause.Occur.SHOULD); @@ -156,7 +154,6 @@ public void testDismaxQuery() throws IOException { } { BooleanQuery.Builder query = new BooleanQuery.Builder(); - query.setDisableCoord(true); DisjunctionMaxQuery uname = new DisjunctionMaxQuery( Arrays.asList(new TermQuery(new Term("username", "foo")), new TermQuery(new Term("song", "foo"))), 0.0f); @@ -186,13 +183,12 @@ public void testBasics() { } String term = TestUtil.randomRealisticUnicodeString(random(), 1, 10); Term[] terms = toTerms(fields, term); - boolean disableCoord = random().nextBoolean(); boolean useBoolean = random().nextBoolean(); float tieBreaker = random().nextFloat(); - BlendedTermQuery query = useBoolean ? BlendedTermQuery.booleanBlendedQuery(terms, disableCoord) : BlendedTermQuery.dismaxBlendedQuery(terms, tieBreaker); + BlendedTermQuery query = useBoolean ? BlendedTermQuery.booleanBlendedQuery(terms) : BlendedTermQuery.dismaxBlendedQuery(terms, tieBreaker); QueryUtils.check(query); terms = toTerms(fields, term); - BlendedTermQuery query2 = useBoolean ? BlendedTermQuery.booleanBlendedQuery(terms, disableCoord) : BlendedTermQuery.dismaxBlendedQuery(terms, tieBreaker); + BlendedTermQuery query2 = useBoolean ? BlendedTermQuery.booleanBlendedQuery(terms) : BlendedTermQuery.dismaxBlendedQuery(terms, tieBreaker); assertEquals(query, query2); } } @@ -222,7 +218,7 @@ public void testExtractTerms() throws IOException { } BlendedTermQuery blendedTermQuery = random().nextBoolean() ? BlendedTermQuery.dismaxBlendedQuery(terms.toArray(new Term[0]), random().nextFloat()) : - BlendedTermQuery.booleanBlendedQuery(terms.toArray(new Term[0]), random().nextBoolean()); + BlendedTermQuery.booleanBlendedQuery(terms.toArray(new Term[0])); Set extracted = new HashSet<>(); IndexSearcher searcher = new IndexSearcher(new MultiReader()); searcher.createNormalizedWeight(blendedTermQuery, false).extractTerms(extracted); diff --git a/core/src/test/java/org/elasticsearch/common/lucene/LuceneTests.java b/core/src/test/java/org/elasticsearch/common/lucene/LuceneTests.java index 33a0b855a7a66..a1132647c7eea 100644 --- a/core/src/test/java/org/elasticsearch/common/lucene/LuceneTests.java +++ b/core/src/test/java/org/elasticsearch/common/lucene/LuceneTests.java @@ -28,7 +28,6 @@ import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; -import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.NoDeletionPolicy; import org.apache.lucene.index.NoMergePolicy; @@ -375,7 +374,7 @@ public void testAsSequentialAccessBits() throws Exception { try (DirectoryReader reader = DirectoryReader.open(w)) { IndexSearcher searcher = newSearcher(reader); - Weight termWeight = new TermQuery(new Term("foo", "bar")).createWeight(searcher, false); + Weight termWeight = new TermQuery(new Term("foo", "bar")).createWeight(searcher, false, 1f); assertEquals(1, reader.leaves().size()); LeafReaderContext leafReaderContext = searcher.getIndexReader().leaves().get(0); Bits bits = Lucene.asSequentialAccessBits(leafReaderContext.reader().maxDoc(), termWeight.scorer(leafReaderContext)); diff --git a/core/src/test/java/org/elasticsearch/common/lucene/ShardCoreKeyMapTests.java b/core/src/test/java/org/elasticsearch/common/lucene/ShardCoreKeyMapTests.java index 5cba06741a997..ca24a1346fe11 100644 --- a/core/src/test/java/org/elasticsearch/common/lucene/ShardCoreKeyMapTests.java +++ b/core/src/test/java/org/elasticsearch/common/lucene/ShardCoreKeyMapTests.java @@ -112,7 +112,7 @@ public void testBasics() throws IOException { assertEquals(3, map.size()); for (LeafReaderContext ctx : reader2.leaves()) { - assertEquals(shardId2, map.getShardId(ctx.reader().getCoreCacheKey())); + assertEquals(shardId2, map.getShardId(ctx.reader().getCoreCacheHelper().getKey())); } w1.addDocument(new Document()); @@ -136,7 +136,7 @@ public void testBasics() throws IOException { final Set index1Keys = new HashSet<>(); for (DirectoryReader reader : Arrays.asList(reader1, reader2)) { for (LeafReaderContext ctx : reader.leaves()) { - index1Keys.add(ctx.reader().getCoreCacheKey()); + index1Keys.add(ctx.reader().getCoreCacheHelper().getKey()); } } index1Keys.removeAll(map.getCoreKeysForIndex("index1")); diff --git a/core/src/test/java/org/elasticsearch/common/lucene/index/ESDirectoryReaderTests.java b/core/src/test/java/org/elasticsearch/common/lucene/index/ESDirectoryReaderTests.java index 817dabfece3f7..397dd28460492 100644 --- a/core/src/test/java/org/elasticsearch/common/lucene/index/ESDirectoryReaderTests.java +++ b/core/src/test/java/org/elasticsearch/common/lucene/index/ESDirectoryReaderTests.java @@ -29,7 +29,6 @@ import org.apache.lucene.index.Term; import org.apache.lucene.store.Directory; import org.apache.lucene.util.IOUtils; -import org.elasticsearch.index.Index; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.test.ESTestCase; @@ -66,7 +65,7 @@ public void testCoreCacheKey() throws Exception { // we should have the same cache key as before assertEquals(1, ir2.numDocs()); assertEquals(1, ir2.leaves().size()); - assertSame(ir.leaves().get(0).reader().getCoreCacheKey(), ir2.leaves().get(0).reader().getCoreCacheKey()); + assertSame(ir.leaves().get(0).reader().getCoreCacheHelper().getKey(), ir2.leaves().get(0).reader().getCoreCacheHelper().getKey()); IOUtils.close(ir, ir2, iw, dir); } } diff --git a/core/src/test/java/org/elasticsearch/deps/lucene/SimpleLuceneTests.java b/core/src/test/java/org/elasticsearch/deps/lucene/SimpleLuceneTests.java index 2b08213ab438d..086a504505973 100644 --- a/core/src/test/java/org/elasticsearch/deps/lucene/SimpleLuceneTests.java +++ b/core/src/test/java/org/elasticsearch/deps/lucene/SimpleLuceneTests.java @@ -21,24 +21,18 @@ import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; -import org.apache.lucene.document.FieldType; -import org.apache.lucene.document.LegacyIntField; +import org.apache.lucene.document.IntPoint; import org.apache.lucene.document.SortedDocValuesField; import org.apache.lucene.document.TextField; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.FieldInfo; -import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.IndexableField; -import org.apache.lucene.index.LeafReader; -import org.apache.lucene.index.PostingsEnum; -import org.apache.lucene.index.SlowCompositeReaderWrapper; +import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.StoredFieldVisitor; import org.apache.lucene.index.Term; -import org.apache.lucene.index.Terms; -import org.apache.lucene.index.TermsEnum; import org.apache.lucene.search.FieldDoc; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.MatchAllDocsQuery; @@ -50,8 +44,6 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.store.RAMDirectory; import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.BytesRefBuilder; -import org.apache.lucene.util.LegacyNumericUtils; import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.test.ESTestCase; @@ -71,7 +63,7 @@ public void testSortValues() throws Exception { document.add(new SortedDocValuesField("str", new BytesRef(text))); indexWriter.addDocument(document); } - IndexReader reader = SlowCompositeReaderWrapper.wrap(DirectoryReader.open(indexWriter)); + IndexReader reader = DirectoryReader.open(indexWriter); IndexSearcher searcher = new IndexSearcher(reader); TopFieldDocs docs = searcher.search(new MatchAllDocsQuery(), 10, new Sort(new SortField("str", SortField.Type.STRING))); for (int i = 0; i < 10; i++) { @@ -86,7 +78,7 @@ public void testSimpleNumericOps() throws Exception { Document document = new Document(); document.add(new TextField("_id", "1", Field.Store.YES)); - document.add(new LegacyIntField("test", 2, LegacyIntField.TYPE_STORED)); + document.add(new IntPoint("test", 2)); indexWriter.addDocument(document); IndexReader reader = DirectoryReader.open(indexWriter); @@ -94,11 +86,9 @@ public void testSimpleNumericOps() throws Exception { TopDocs topDocs = searcher.search(new TermQuery(new Term("_id", "1")), 1); Document doc = searcher.doc(topDocs.scoreDocs[0].doc); IndexableField f = doc.getField("test"); - assertThat(f.stringValue(), equalTo("2")); + assertThat(f.numericValue(), equalTo(2)); - BytesRefBuilder bytes = new BytesRefBuilder(); - LegacyNumericUtils.intToPrefixCoded(2, 0, bytes); - topDocs = searcher.search(new TermQuery(new Term("test", bytes.get())), 1); + topDocs = searcher.search(IntPoint.newExactQuery("test", 2), 1); doc = searcher.doc(topDocs.scoreDocs[0].doc); f = doc.getField("test"); assertThat(f.stringValue(), equalTo("2")); @@ -139,40 +129,6 @@ public Status needsField(FieldInfo fieldInfo) throws IOException { indexWriter.close(); } - public void testBoost() throws Exception { - Directory dir = new RAMDirectory(); - IndexWriter indexWriter = new IndexWriter(dir, new IndexWriterConfig(Lucene.STANDARD_ANALYZER)); - - for (int i = 0; i < 100; i++) { - // TODO (just setting the boost value does not seem to work...) - StringBuilder value = new StringBuilder().append("value"); - for (int j = 0; j < i; j++) { - value.append(" ").append("value"); - } - Document document = new Document(); - TextField textField = new TextField("_id", Integer.toString(i), Field.Store.YES); - textField.setBoost(i); - document.add(textField); - textField = new TextField("value", value.toString(), Field.Store.YES); - textField.setBoost(i); - document.add(textField); - indexWriter.addDocument(document); - } - - IndexReader reader = DirectoryReader.open(indexWriter); - IndexSearcher searcher = new IndexSearcher(reader); - TermQuery query = new TermQuery(new Term("value", "value")); - TopDocs topDocs = searcher.search(query, 100); - assertThat(100, equalTo(topDocs.totalHits)); - for (int i = 0; i < topDocs.scoreDocs.length; i++) { - Document doc = searcher.doc(topDocs.scoreDocs[i].doc); -// System.out.println(doc.get("id") + ": " + searcher.explain(query, topDocs.scoreDocs[i].doc)); - assertThat(doc.get("_id"), equalTo(Integer.toString(100 - i - 1))); - } - - indexWriter.close(); - } - public void testNRTSearchOnClosedWriter() throws Exception { Directory dir = new RAMDirectory(); IndexWriter indexWriter = new IndexWriter(dir, new IndexWriterConfig(Lucene.STANDARD_ANALYZER)); @@ -181,7 +137,6 @@ public void testNRTSearchOnClosedWriter() throws Exception { for (int i = 0; i < 100; i++) { Document document = new Document(); TextField field = new TextField("_id", Integer.toString(i), Field.Store.YES); - field.setBoost(i); document.add(field); indexWriter.addDocument(document); } @@ -189,60 +144,9 @@ public void testNRTSearchOnClosedWriter() throws Exception { indexWriter.close(); - TermsEnum termDocs = SlowCompositeReaderWrapper.wrap(reader).terms("_id").iterator(); - termDocs.next(); - } - - /** - * A test just to verify that term freqs are not stored for numeric fields. int1 is not storing termFreq - * and int2 does. - */ - public void testNumericTermDocsFreqs() throws Exception { - Directory dir = new RAMDirectory(); - IndexWriter indexWriter = new IndexWriter(dir, new IndexWriterConfig(Lucene.STANDARD_ANALYZER)); - - Document doc = new Document(); - FieldType type = LegacyIntField.TYPE_NOT_STORED; - LegacyIntField field = new LegacyIntField("int1", 1, type); - doc.add(field); - - type = new FieldType(LegacyIntField.TYPE_NOT_STORED); - type.setIndexOptions(IndexOptions.DOCS_AND_FREQS); - type.freeze(); - - field = new LegacyIntField("int1", 1, type); - doc.add(field); - - field = new LegacyIntField("int2", 1, type); - doc.add(field); - - field = new LegacyIntField("int2", 1, type); - doc.add(field); - - indexWriter.addDocument(doc); - - IndexReader reader = DirectoryReader.open(indexWriter); - LeafReader atomicReader = SlowCompositeReaderWrapper.wrap(reader); - - Terms terms = atomicReader.terms("int1"); - TermsEnum termsEnum = terms.iterator(); - termsEnum.next(); - - PostingsEnum termDocs = termsEnum.postings(null); - assertThat(termDocs.nextDoc(), equalTo(0)); - assertThat(termDocs.docID(), equalTo(0)); - assertThat(termDocs.freq(), equalTo(1)); - - terms = atomicReader.terms("int2"); - termsEnum = terms.iterator(); - termsEnum.next(); - termDocs = termsEnum.postings(termDocs); - assertThat(termDocs.nextDoc(), equalTo(0)); - assertThat(termDocs.docID(), equalTo(0)); - assertThat(termDocs.freq(), equalTo(2)); - - reader.close(); - indexWriter.close(); + for (LeafReaderContext leaf : reader.leaves()) { + leaf.reader().terms("_id").iterator().next(); + } } private DirectoryReader refreshReader(DirectoryReader reader) throws IOException { diff --git a/core/src/test/java/org/elasticsearch/index/IndexModuleTests.java b/core/src/test/java/org/elasticsearch/index/IndexModuleTests.java index 106dc9274da93..6f80da2d639c7 100644 --- a/core/src/test/java/org/elasticsearch/index/IndexModuleTests.java +++ b/core/src/test/java/org/elasticsearch/index/IndexModuleTests.java @@ -431,8 +431,8 @@ public long computeNorm(FieldInvertState state) { } @Override - public SimWeight computeWeight(CollectionStatistics collectionStats, TermStatistics... termStats) { - return delegate.computeWeight(collectionStats, termStats); + public SimWeight computeWeight(float boost, CollectionStatistics collectionStats, TermStatistics... termStats) { + return delegate.computeWeight(boost, collectionStats, termStats); } @Override diff --git a/core/src/test/java/org/elasticsearch/index/codec/CodecTests.java b/core/src/test/java/org/elasticsearch/index/codec/CodecTests.java index e49e21bd2ff99..57dfd19185dbb 100644 --- a/core/src/test/java/org/elasticsearch/index/codec/CodecTests.java +++ b/core/src/test/java/org/elasticsearch/index/codec/CodecTests.java @@ -20,12 +20,10 @@ package org.elasticsearch.index.codec; import org.apache.lucene.codecs.Codec; -import org.apache.lucene.codecs.lucene50.Lucene50Codec; import org.apache.lucene.codecs.lucene50.Lucene50StoredFieldsFormat; import org.apache.lucene.codecs.lucene50.Lucene50StoredFieldsFormat.Mode; -import org.apache.lucene.codecs.lucene53.Lucene53Codec; -import org.apache.lucene.codecs.lucene54.Lucene54Codec; import org.apache.lucene.codecs.lucene62.Lucene62Codec; +import org.apache.lucene.codecs.lucene70.Lucene70Codec; import org.apache.lucene.document.Document; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexWriter; @@ -55,10 +53,8 @@ public class CodecTests extends ESTestCase { public void testResolveDefaultCodecs() throws Exception { CodecService codecService = createCodecService(); assertThat(codecService.codec("default"), instanceOf(PerFieldMappingPostingFormatCodec.class)); - assertThat(codecService.codec("default"), instanceOf(Lucene62Codec.class)); - assertThat(codecService.codec("Lucene54"), instanceOf(Lucene54Codec.class)); - assertThat(codecService.codec("Lucene53"), instanceOf(Lucene53Codec.class)); - assertThat(codecService.codec("Lucene50"), instanceOf(Lucene50Codec.class)); + assertThat(codecService.codec("default"), instanceOf(Lucene70Codec.class)); + assertThat(codecService.codec("Lucene62"), instanceOf(Lucene62Codec.class)); } public void testDefault() throws Exception { diff --git a/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java b/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java index 65bebc40933b1..db5d15cdd2151 100644 --- a/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java +++ b/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java @@ -1926,7 +1926,10 @@ private static FixedBitSet getSeqNosSet(final IndexReader reader, final long hig final Bits bits = leaf.getLiveDocs(); for (int docID = 0; docID < leaf.maxDoc(); docID++) { if (bits == null || bits.get(docID)) { - final long seqNo = values.get(docID); + if (values.advanceExact(docID) == false) { + throw new AssertionError("Document does not have a seq number: " + docID); + } + final long seqNo = values.longValue(); assertFalse("should not have more than one document with the same seq_no[" + seqNo + "]", bitSet.get((int) seqNo)); bitSet.set((int) seqNo); } diff --git a/core/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java index 9c67292431752..e6a1c0a69d81a 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java @@ -97,9 +97,9 @@ public void testDefaults() throws IOException { assertEquals(new BytesRef("T"), leaf.terms("field").iterator().next()); SortedNumericDocValues values = leaf.getSortedNumericDocValues("field"); assertNotNull(values); - values.setDocument(0); - assertEquals(1, values.count()); - assertEquals(1, values.valueAt(0)); + assertTrue(values.advanceExact(0)); + assertEquals(1, values.docValueCount()); + assertEquals(1, values.nextValue()); } } } diff --git a/core/src/test/java/org/elasticsearch/index/mapper/FakeStringFieldMapper.java b/core/src/test/java/org/elasticsearch/index/mapper/FakeStringFieldMapper.java index 2969b8392b5f2..642282c9d5c62 100755 --- a/core/src/test/java/org/elasticsearch/index/mapper/FakeStringFieldMapper.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/FakeStringFieldMapper.java @@ -121,11 +121,6 @@ protected FakeStringFieldMapper(String simpleName, FakeStringFieldType fieldType super(simpleName, fieldType, defaultFieldType, indexSettings, multiFields, copyTo); } - @Override - protected boolean customBoost() { - return true; - } - @Override protected void parseCreateField(ParseContext context, List fields) throws IOException { String value; diff --git a/core/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldTypeTests.java b/core/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldTypeTests.java index dd66421986779..1945c164fc493 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldTypeTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldTypeTests.java @@ -195,19 +195,19 @@ public void testFieldData() throws IOException { assertEquals(fielddata.getNumericType(), IndexNumericFieldData.NumericType.DOUBLE); AtomicNumericFieldData leafFieldData = fielddata.load(reader.leaves().get(0)); SortedNumericDoubleValues values = leafFieldData.getDoubleValues(); - values.setDocument(0); - assertEquals(1, values.count()); - assertEquals(10/ft.getScalingFactor(), values.valueAt(0), 10e-5); + assertTrue(values.advanceExact(0)); + assertEquals(1, values.docValueCount()); + assertEquals(10/ft.getScalingFactor(), values.nextValue(), 10e-5); // multi-valued ft.setName("scaled_float2"); fielddata = (IndexNumericFieldData) ft.fielddataBuilder().build(indexSettings, ft, null, null, null); leafFieldData = fielddata.load(reader.leaves().get(0)); values = leafFieldData.getDoubleValues(); - values.setDocument(0); - assertEquals(2, values.count()); - assertEquals(5/ft.getScalingFactor(), values.valueAt(0), 10e-5); - assertEquals(12/ft.getScalingFactor(), values.valueAt(1), 10e-5); + assertTrue(values.advanceExact(0)); + assertEquals(2, values.docValueCount()); + assertEquals(5/ft.getScalingFactor(), values.nextValue(), 10e-5); + assertEquals(12/ft.getScalingFactor(), values.nextValue(), 10e-5); } IOUtils.close(w, dir); } diff --git a/core/src/test/java/org/elasticsearch/index/query/MultiMatchQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/MultiMatchQueryBuilderTests.java index 63fa1eea01734..195c3aa1dfc11 100644 --- a/core/src/test/java/org/elasticsearch/index/query/MultiMatchQueryBuilderTests.java +++ b/core/src/test/java/org/elasticsearch/index/query/MultiMatchQueryBuilderTests.java @@ -26,7 +26,6 @@ import org.apache.lucene.search.DisjunctionMaxQuery; import org.apache.lucene.search.FuzzyQuery; import org.apache.lucene.search.IndexOrDocValuesQuery; -import org.apache.lucene.search.LegacyNumericRangeQuery; import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.PhraseQuery; @@ -151,7 +150,6 @@ protected void doAssertLuceneQuery(MultiMatchQueryBuilder queryBuilder, Query qu .or(instanceOf(FuzzyQuery.class)).or(instanceOf(MultiPhrasePrefixQuery.class)) .or(instanceOf(MatchAllDocsQuery.class)).or(instanceOf(ExtendedCommonTermsQuery.class)) .or(instanceOf(MatchNoDocsQuery.class)).or(instanceOf(PhraseQuery.class)) - .or(instanceOf(LegacyNumericRangeQuery.class)) .or(instanceOf(PointRangeQuery.class)).or(instanceOf(IndexOrDocValuesQuery.class))); } diff --git a/core/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java index 38e23bd5d502c..3c63d2c6503c0 100644 --- a/core/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java +++ b/core/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java @@ -374,7 +374,6 @@ public void testToQueryWilcardQueryWithSynonyms() throws Exception { BooleanClause.Occur.SHOULD)) .add(new BooleanClause(new PrefixQuery(new Term(STRING_FIELD_NAME, "foobar")), BooleanClause.Occur.SHOULD)) - .setDisableCoord(true) .build(), defaultOp) .build(), defaultOp) .add(new BooleanClause(new SynonymQuery(new Term(STRING_FIELD_NAME, "last"), @@ -463,7 +462,6 @@ public void testToQueryWithGraph() throws Exception { new SpanTermQuery(new Term(STRING_FIELD_NAME, "cavy")))) .addClause(new SpanTermQuery(new Term(STRING_FIELD_NAME, "smells"))) .build(), Occur.SHOULD) - .setDisableCoord(true) .build(); assertThat(query, Matchers.equalTo(expectedQuery)); @@ -481,7 +479,6 @@ public void testToQueryWithGraph() throws Exception { .setSlop(2) .build(), Occur.SHOULD) - .setDisableCoord(true) .build(); assertThat(query, Matchers.equalTo(expectedQuery)); } diff --git a/core/src/test/java/org/elasticsearch/index/query/SimpleQueryParserTests.java b/core/src/test/java/org/elasticsearch/index/query/SimpleQueryParserTests.java index de423db442a1b..53ba0f6d2b651 100644 --- a/core/src/test/java/org/elasticsearch/index/query/SimpleQueryParserTests.java +++ b/core/src/test/java/org/elasticsearch/index/query/SimpleQueryParserTests.java @@ -108,7 +108,6 @@ public void testAnalyzerWildcardWithSynonyms() { BooleanClause.Occur.SHOULD)) .add(new BooleanClause(new PrefixQuery(new Term("field1", "foobar")), BooleanClause.Occur.SHOULD)) - .setDisableCoord(true) .build(), defaultOp) .build(), defaultOp) .add(new BooleanClause(new SynonymQuery(new Term("field1", "last"), diff --git a/core/src/test/java/org/elasticsearch/index/query/functionscore/FunctionScoreTests.java b/core/src/test/java/org/elasticsearch/index/query/functionscore/FunctionScoreTests.java index 16e66f50e0d1c..341f4d87bdceb 100644 --- a/core/src/test/java/org/elasticsearch/index/query/functionscore/FunctionScoreTests.java +++ b/core/src/test/java/org/elasticsearch/index/query/functionscore/FunctionScoreTests.java @@ -99,16 +99,17 @@ public ScriptDocValues getScriptValues() { public SortedBinaryDocValues getBytesValues() { return new SortedBinaryDocValues() { @Override - public void setDocument(int docId) { + public boolean advanceExact(int docId) { + return true; } @Override - public int count() { + public int docValueCount() { return 1; } @Override - public BytesRef valueAt(int index) { + public BytesRef nextValue() { return new BytesRef("0"); } }; @@ -178,17 +179,18 @@ public SortedNumericDocValues getLongValues() { public SortedNumericDoubleValues getDoubleValues() { return new SortedNumericDoubleValues() { @Override - public void setDocument(int doc) { + public boolean advanceExact(int docId) { + return true; } @Override - public double valueAt(int index) { + public int docValueCount() { return 1; } @Override - public int count() { - return 1; + public double nextValue() { + return 1d; } }; } @@ -614,7 +616,7 @@ public void testPropagatesApproximations() throws IOException { FunctionScoreQuery fsq = new FunctionScoreQuery(query, null, null, null, Float.POSITIVE_INFINITY); for (boolean needsScores : new boolean[] {true, false}) { - Weight weight = searcher.createWeight(fsq, needsScores); + Weight weight = searcher.createWeight(fsq, needsScores, 1f); Scorer scorer = weight.scorer(reader.leaves().get(0)); assertNotNull(scorer.twoPhaseIterator()); } @@ -622,7 +624,7 @@ public void testPropagatesApproximations() throws IOException { FiltersFunctionScoreQuery ffsq = new FiltersFunctionScoreQuery(query, ScoreMode.SUM, new FilterFunction[0], Float.POSITIVE_INFINITY, null, CombineFunction.MULTIPLY); for (boolean needsScores : new boolean[] {true, false}) { - Weight weight = searcher.createWeight(ffsq, needsScores); + Weight weight = searcher.createWeight(ffsq, needsScores, 1f); Scorer scorer = weight.scorer(reader.leaves().get(0)); assertNotNull(scorer.twoPhaseIterator()); } diff --git a/core/src/test/java/org/elasticsearch/index/query/plugin/DummyQueryParserPlugin.java b/core/src/test/java/org/elasticsearch/index/query/plugin/DummyQueryParserPlugin.java index 86c03c7273883..3d0eee79595f5 100644 --- a/core/src/test/java/org/elasticsearch/index/query/plugin/DummyQueryParserPlugin.java +++ b/core/src/test/java/org/elasticsearch/index/query/plugin/DummyQueryParserPlugin.java @@ -52,8 +52,8 @@ public String toString(String field) { } @Override - public Weight createWeight(IndexSearcher searcher, boolean needsScores) throws IOException { - return matchAllDocsQuery.createWeight(searcher, needsScores); + public Weight createWeight(IndexSearcher searcher, boolean needsScores, float boost) throws IOException { + return matchAllDocsQuery.createWeight(searcher, needsScores, boost); } @Override diff --git a/core/src/test/java/org/elasticsearch/index/search/MultiMatchQueryTests.java b/core/src/test/java/org/elasticsearch/index/search/MultiMatchQueryTests.java index 992667a5056e8..be6b0c3865aeb 100644 --- a/core/src/test/java/org/elasticsearch/index/search/MultiMatchQueryTests.java +++ b/core/src/test/java/org/elasticsearch/index/search/MultiMatchQueryTests.java @@ -110,7 +110,7 @@ public void testBlendTerms() { ft2.setName("bar"); Term[] terms = new Term[] { new Term("foo", "baz"), new Term("bar", "baz") }; float[] boosts = new float[] {2, 3}; - Query expected = BlendedTermQuery.booleanBlendedQuery(terms, boosts, false); + Query expected = BlendedTermQuery.booleanBlendedQuery(terms, boosts); Query actual = MultiMatchQuery.blendTerm( indexService.newQueryShardContext(randomInt(20), null, () -> { throw new UnsupportedOperationException(); }), new BytesRef("baz"), null, 1f, new FieldAndFieldType(ft1, 2), new FieldAndFieldType(ft2, 3)); @@ -126,7 +126,7 @@ public void testBlendTermsWithFieldBoosts() { ft2.setBoost(10); Term[] terms = new Term[] { new Term("foo", "baz"), new Term("bar", "baz") }; float[] boosts = new float[] {200, 30}; - Query expected = BlendedTermQuery.booleanBlendedQuery(terms, boosts, false); + Query expected = BlendedTermQuery.booleanBlendedQuery(terms, boosts); Query actual = MultiMatchQuery.blendTerm( indexService.newQueryShardContext(randomInt(20), null, () -> { throw new UnsupportedOperationException(); }), new BytesRef("baz"), null, 1f, new FieldAndFieldType(ft1, 2), new FieldAndFieldType(ft2, 3)); @@ -145,7 +145,7 @@ public Query termQuery(Object value, QueryShardContext context) { ft2.setName("bar"); Term[] terms = new Term[] { new Term("foo", "baz") }; float[] boosts = new float[] {2}; - Query expected = BlendedTermQuery.booleanBlendedQuery(terms, boosts, false); + Query expected = BlendedTermQuery.booleanBlendedQuery(terms, boosts); Query actual = MultiMatchQuery.blendTerm( indexService.newQueryShardContext(randomInt(20), null, () -> { throw new UnsupportedOperationException(); }), new BytesRef("baz"), null, 1f, new FieldAndFieldType(ft1, 2), new FieldAndFieldType(ft2, 3)); @@ -164,9 +164,9 @@ public Query termQuery(Object value, QueryShardContext context) { ft2.setName("bar"); Term[] terms = new Term[] { new Term("foo", "baz") }; float[] boosts = new float[] {2}; - Query expectedClause1 = BlendedTermQuery.booleanBlendedQuery(terms, boosts, false); + Query expectedClause1 = BlendedTermQuery.booleanBlendedQuery(terms, boosts); Query expectedClause2 = new BoostQuery(new MatchAllDocsQuery(), 3); - Query expected = new BooleanQuery.Builder().setDisableCoord(true) + Query expected = new BooleanQuery.Builder() .add(expectedClause1, Occur.SHOULD) .add(expectedClause2, Occur.SHOULD) .build(); diff --git a/core/src/test/java/org/elasticsearch/index/shard/IndexSearcherWrapperTests.java b/core/src/test/java/org/elasticsearch/index/shard/IndexSearcherWrapperTests.java index 6b7c77fdf5e19..f5d4f048c27f5 100644 --- a/core/src/test/java/org/elasticsearch/index/shard/IndexSearcherWrapperTests.java +++ b/core/src/test/java/org/elasticsearch/index/shard/IndexSearcherWrapperTests.java @@ -76,8 +76,8 @@ public IndexSearcher wrap(IndexSearcher searcher) throws EngineException { try (Engine.Searcher engineSearcher = new Engine.Searcher("foo", searcher)) { final Engine.Searcher wrap = wrapper.wrap(engineSearcher); assertEquals(1, wrap.reader().getRefCount()); - ElasticsearchDirectoryReader.addReaderCloseListener(wrap.getDirectoryReader(), reader -> { - if (reader == open) { + ElasticsearchDirectoryReader.addReaderCloseListener(wrap.getDirectoryReader(), key -> { + if (key == open.getReaderCacheHelper().getKey()) { count.incrementAndGet(); } outerCount.incrementAndGet(); @@ -123,11 +123,11 @@ public IndexSearcher wrap(IndexSearcher searcher) throws EngineException { final ConcurrentHashMap cache = new ConcurrentHashMap<>(); try (Engine.Searcher engineSearcher = new Engine.Searcher("foo", searcher)) { try (Engine.Searcher wrap = wrapper.wrap(engineSearcher)) { - ElasticsearchDirectoryReader.addReaderCloseListener(wrap.getDirectoryReader(), reader -> { - cache.remove(reader.getCoreCacheKey()); + ElasticsearchDirectoryReader.addReaderCloseListener(wrap.getDirectoryReader(), key -> { + cache.remove(key); }); TopDocs search = wrap.searcher().search(new TermQuery(new Term("field", "doc")), 1); - cache.put(wrap.reader().getCoreCacheKey(), search); + cache.put(wrap.reader().getReaderCacheHelper().getKey(), search); } } assertEquals(1, closeCalls.get()); @@ -158,49 +158,6 @@ public void testNoWrap() throws IOException { IOUtils.close(open, writer, dir); } - public void testWrappedReaderMustDelegateCoreCacheKey() throws IOException { - Directory dir = newDirectory(); - IndexWriterConfig iwc = newIndexWriterConfig(); - IndexWriter writer = new IndexWriter(dir, iwc); - Document doc = new Document(); - doc.add(new StringField("id", "1", random().nextBoolean() ? Field.Store.YES : Field.Store.NO)); - doc.add(new TextField("field", "doc", random().nextBoolean() ? Field.Store.YES : Field.Store.NO)); - writer.addDocument(doc); - DirectoryReader open = ElasticsearchDirectoryReader.wrap(DirectoryReader.open(writer), new ShardId("foo", "_na_", 1)); - IndexSearcher searcher = new IndexSearcher(open); - assertEquals(1, searcher.search(new TermQuery(new Term("field", "doc")), 1).totalHits); - searcher.setSimilarity(iwc.getSimilarity()); - IndexSearcherWrapper wrapper = new IndexSearcherWrapper() { - @Override - protected DirectoryReader wrap(DirectoryReader reader) throws IOException { - return new BrokenWrapper(reader, false); - } - }; - try (Engine.Searcher engineSearcher = new Engine.Searcher("foo", searcher)) { - try { - wrapper.wrap(engineSearcher); - fail("reader must delegate cache key"); - } catch (IllegalStateException ex) { - // all is well - } - } - wrapper = new IndexSearcherWrapper() { - @Override - protected DirectoryReader wrap(DirectoryReader reader) throws IOException { - return new BrokenWrapper(reader, true); - } - }; - try (Engine.Searcher engineSearcher = new Engine.Searcher("foo", searcher)) { - try { - wrapper.wrap(engineSearcher); - fail("reader must delegate cache key"); - } catch (IllegalStateException ex) { - // all is well - } - } - IOUtils.close(open, writer, dir); - } - private static class FieldMaskingReader extends FilterDirectoryReader { private final String field; private final AtomicInteger closeCalls; @@ -222,8 +179,8 @@ protected DirectoryReader doWrapDirectoryReader(DirectoryReader in) throws IOExc } @Override - public Object getCoreCacheKey() { - return in.getCoreCacheKey(); + public CacheHelper getReaderCacheHelper() { + return in.getReaderCacheHelper(); } @Override @@ -233,44 +190,4 @@ protected void doClose() throws IOException { } } - private static class BrokenWrapper extends FilterDirectoryReader { - - private final boolean hideDelegate; - - BrokenWrapper(DirectoryReader in, boolean hideDelegate) throws IOException { - super(in, new SubReaderWrapper() { - @Override - public LeafReader wrap(LeafReader reader) { - return reader; - } - }); - this.hideDelegate = hideDelegate; - } - - @Override - protected DirectoryReader doWrapDirectoryReader(DirectoryReader in) throws IOException { - return new BrokenWrapper(in, hideDelegate); - } - - @Override - public DirectoryReader getDelegate() { - if (hideDelegate) { - try { - return ElasticsearchDirectoryReader.wrap(super.getDelegate(), new ShardId("foo", "_na_", 1)); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - return super.getDelegate(); - } - - @Override - public Object getCoreCacheKey() { - if (hideDelegate == false) { - return super.getCoreCacheKey(); - } else { - return in.getCoreCacheKey(); - } - } - } } diff --git a/core/src/test/java/org/elasticsearch/index/store/StoreTests.java b/core/src/test/java/org/elasticsearch/index/store/StoreTests.java index 156c2132358f5..8575c4e3e6afb 100644 --- a/core/src/test/java/org/elasticsearch/index/store/StoreTests.java +++ b/core/src/test/java/org/elasticsearch/index/store/StoreTests.java @@ -858,7 +858,7 @@ public void testUserDataRead() throws IOException { String translogId = "a translog id"; commitData.put(Engine.SYNC_COMMIT_ID, syncId); commitData.put(Translog.TRANSLOG_GENERATION_KEY, translogId); - writer.setCommitData(commitData); + writer.setLiveCommitData(commitData.entrySet()); writer.commit(); writer.close(); Store.MetadataSnapshot metadata; diff --git a/core/src/test/java/org/elasticsearch/indices/IndicesQueryCacheTests.java b/core/src/test/java/org/elasticsearch/indices/IndicesQueryCacheTests.java index cd94ee0f8e954..10f098787c027 100644 --- a/core/src/test/java/org/elasticsearch/indices/IndicesQueryCacheTests.java +++ b/core/src/test/java/org/elasticsearch/indices/IndicesQueryCacheTests.java @@ -69,9 +69,9 @@ public String toString(String field) { } @Override - public Weight createWeight(IndexSearcher searcher, boolean needsScores) + public Weight createWeight(IndexSearcher searcher, boolean needsScores, float boost) throws IOException { - return new ConstantScoreWeight(this) { + return new ConstantScoreWeight(this, boost) { @Override public Scorer scorer(LeafReaderContext context) throws IOException { return new ConstantScoreScorer(this, score(), DocIdSetIterator.all(context.reader().maxDoc())); diff --git a/core/src/test/java/org/elasticsearch/indices/memory/breaker/RandomExceptionCircuitBreakerIT.java b/core/src/test/java/org/elasticsearch/indices/memory/breaker/RandomExceptionCircuitBreakerIT.java index 2f336d8d69cd1..6dd9126a51983 100644 --- a/core/src/test/java/org/elasticsearch/indices/memory/breaker/RandomExceptionCircuitBreakerIT.java +++ b/core/src/test/java/org/elasticsearch/indices/memory/breaker/RandomExceptionCircuitBreakerIT.java @@ -41,7 +41,6 @@ import org.elasticsearch.indices.IndicesService; import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache; import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.search.basic.SearchWithRandomExceptionsIT; import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.engine.MockEngineSupport; @@ -298,5 +297,10 @@ public RandomExceptionDirectoryReaderWrapper(DirectoryReader in, Settings settin protected DirectoryReader doWrapDirectoryReader(DirectoryReader in) throws IOException { return new RandomExceptionDirectoryReaderWrapper(in, settings); } + + @Override + public CacheHelper getReaderCacheHelper() { + return in.getReaderCacheHelper(); + } } } diff --git a/core/src/test/java/org/elasticsearch/search/MultiValueModeTests.java b/core/src/test/java/org/elasticsearch/search/MultiValueModeTests.java index 5caba0fb44139..d33e0543f5cba 100644 --- a/core/src/test/java/org/elasticsearch/search/MultiValueModeTests.java +++ b/core/src/test/java/org/elasticsearch/search/MultiValueModeTests.java @@ -20,17 +20,23 @@ package org.elasticsearch.search; import com.carrotsearch.randomizedtesting.generators.RandomStrings; + import org.apache.lucene.index.BinaryDocValues; import org.apache.lucene.index.DocValues; import org.apache.lucene.index.NumericDocValues; -import org.apache.lucene.index.RandomAccessOrds; import org.apache.lucene.index.SortedDocValues; import org.apache.lucene.index.SortedNumericDocValues; +import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.util.BitSetIterator; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.FixedBitSet; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.index.fielddata.AbstractBinaryDocValues; +import org.elasticsearch.index.fielddata.AbstractNumericDocValues; +import org.elasticsearch.index.fielddata.AbstractSortedDocValues; +import org.elasticsearch.index.fielddata.AbstractSortedNumericDocValues; +import org.elasticsearch.index.fielddata.AbstractSortedSetDocValues; import org.elasticsearch.index.fielddata.FieldData; import org.elasticsearch.index.fielddata.NumericDoubleValues; import org.elasticsearch.index.fielddata.SortedBinaryDocValues; @@ -80,13 +86,19 @@ public void testSingleValuedLongs() throws Exception { docsWithValue.set(i); } } - final NumericDocValues singleValues = new NumericDocValues() { + final NumericDocValues singleValues = new AbstractNumericDocValues() { + int docId; @Override - public long get(int docID) { - return array[docID]; + public boolean advanceExact(int target) throws IOException { + this.docId = target; + return docsWithValue == null ? true : docsWithValue.get(docId); + } + @Override + public long longValue() { + return array[docId]; } }; - final SortedNumericDocValues multiValues = DocValues.singleton(singleValues, docsWithValue); + final SortedNumericDocValues multiValues = DocValues.singleton(singleValues); verify(multiValues, numDocs); final FixedBitSet rootDocs = randomRootDocs(numDocs); final FixedBitSet innerDocs = randomInnerDocs(rootDocs); @@ -104,21 +116,24 @@ public void testMultiValuedLongs() throws Exception { Arrays.sort(values); array[i] = values; } - final SortedNumericDocValues multiValues = new SortedNumericDocValues() { + final SortedNumericDocValues multiValues = new AbstractSortedNumericDocValues() { int doc; + int i; @Override - public long valueAt(int index) { - return array[doc][index]; + public long nextValue() { + return array[doc][i++]; } @Override - public void setDocument(int doc) { + public boolean advanceExact(int doc) { this.doc = doc; + i = 0; + return array[doc].length > 0; } @Override - public int count() { + public int docValueCount() { return array[doc].length; } }; @@ -128,18 +143,18 @@ public int count() { verify(multiValues, numDocs, rootDocs, innerDocs); } - private void verify(SortedNumericDocValues values, int maxDoc) { + private void verify(SortedNumericDocValues values, int maxDoc) throws IOException { for (long missingValue : new long[] { 0, randomLong() }) { for (MultiValueMode mode : MultiValueMode.values()) { final NumericDocValues selected = mode.select(values, missingValue); for (int i = 0; i < maxDoc; ++i) { - final long actual = selected.get(i); + assertTrue(selected.advanceExact(i)); + final long actual = selected.longValue(); long expected = 0; - values.setDocument(i); - int numValues = values.count(); - if (numValues == 0) { + if (values.advanceExact(i) == false) { expected = missingValue; } else { + int numValues = values.docValueCount(); if (mode == MultiValueMode.MAX) { expected = Long.MIN_VALUE; } else if (mode == MultiValueMode.MIN) { @@ -147,11 +162,11 @@ private void verify(SortedNumericDocValues values, int maxDoc) { } for (int j = 0; j < numValues; ++j) { if (mode == MultiValueMode.SUM || mode == MultiValueMode.AVG) { - expected += values.valueAt(j); + expected += values.nextValue(); } else if (mode == MultiValueMode.MIN) { - expected = Math.min(expected, values.valueAt(j)); + expected = Math.min(expected, values.nextValue()); } else if (mode == MultiValueMode.MAX) { - expected = Math.max(expected, values.valueAt(j)); + expected = Math.max(expected, values.nextValue()); } } if (mode == MultiValueMode.AVG) { @@ -159,9 +174,15 @@ private void verify(SortedNumericDocValues values, int maxDoc) { } else if (mode == MultiValueMode.MEDIAN) { int value = numValues/2; if (numValues % 2 == 0) { - expected = Math.round((values.valueAt(value - 1) + values.valueAt(value))/2.0); + for (int j = 0; j < value - 1; ++j) { + values.nextValue(); + } + expected = Math.round((values.nextValue() + values.nextValue())/2.0); } else { - expected = values.valueAt(value); + for (int j = 0; j < value; ++j) { + values.nextValue(); + } + expected = values.nextValue(); } } } @@ -178,7 +199,8 @@ private void verify(SortedNumericDocValues values, int maxDoc, FixedBitSet rootD final NumericDocValues selected = mode.select(values, missingValue, rootDocs, new BitSetIterator(innerDocs, 0L), maxDoc); int prevRoot = -1; for (int root = rootDocs.nextSetBit(0); root != -1; root = root + 1 < maxDoc ? rootDocs.nextSetBit(root + 1) : -1) { - final long actual = selected.get(root); + assertTrue(selected.advanceExact(root)); + final long actual = selected.longValue(); long expected = 0; if (mode == MultiValueMode.MAX) { expected = Long.MIN_VALUE; @@ -187,16 +209,17 @@ private void verify(SortedNumericDocValues values, int maxDoc, FixedBitSet rootD } int numValues = 0; for (int child = innerDocs.nextSetBit(prevRoot + 1); child != -1 && child < root; child = innerDocs.nextSetBit(child + 1)) { - values.setDocument(child); - for (int j = 0; j < values.count(); ++j) { - if (mode == MultiValueMode.SUM || mode == MultiValueMode.AVG) { - expected += values.valueAt(j); - } else if (mode == MultiValueMode.MIN) { - expected = Math.min(expected, values.valueAt(j)); - } else if (mode == MultiValueMode.MAX) { - expected = Math.max(expected, values.valueAt(j)); + if (values.advanceExact(child)) { + for (int j = 0; j < values.docValueCount(); ++j) { + if (mode == MultiValueMode.SUM || mode == MultiValueMode.AVG) { + expected += values.nextValue(); + } else if (mode == MultiValueMode.MIN) { + expected = Math.min(expected, values.nextValue()); + } else if (mode == MultiValueMode.MAX) { + expected = Math.max(expected, values.nextValue()); + } + ++numValues; } - ++numValues; } } if (numValues == 0) { @@ -228,12 +251,18 @@ public void testSingleValuedDoubles() throws Exception { } } final NumericDoubleValues singleValues = new NumericDoubleValues() { + int docID; @Override - public double get(int docID) { + public boolean advanceExact(int doc) throws IOException { + docID = doc; + return docsWithValue == null || docsWithValue.get(doc); + } + @Override + public double doubleValue() { return array[docID]; } }; - final SortedNumericDoubleValues multiValues = FieldData.singleton(singleValues, docsWithValue); + final SortedNumericDoubleValues multiValues = FieldData.singleton(singleValues); verify(multiValues, numDocs); final FixedBitSet rootDocs = randomRootDocs(numDocs); final FixedBitSet innerDocs = randomInnerDocs(rootDocs); @@ -253,19 +282,22 @@ public void testMultiValuedDoubles() throws Exception { } final SortedNumericDoubleValues multiValues = new SortedNumericDoubleValues() { int doc; + int i; @Override - public double valueAt(int index) { - return array[doc][index]; + public double nextValue() { + return array[doc][i++]; } @Override - public void setDocument(int doc) { + public boolean advanceExact(int doc) { this.doc = doc; + i = 0; + return array[doc].length > 0; } @Override - public int count() { + public int docValueCount() { return array[doc].length; } }; @@ -275,7 +307,7 @@ public int count() { verify(multiValues, numDocs, rootDocs, innerDocs); } - private void verify(SortedNumericDoubleValues values, int maxDoc) { + private void verify(SortedNumericDoubleValues values, int maxDoc) throws IOException { for (long missingValue : new long[] { 0, randomLong() }) { for (MultiValueMode mode : MultiValueMode.values()) { if (MultiValueMode.MEDIAN.equals(mode)) { @@ -283,13 +315,13 @@ private void verify(SortedNumericDoubleValues values, int maxDoc) { } final NumericDoubleValues selected = mode.select(values, missingValue); for (int i = 0; i < maxDoc; ++i) { - final double actual = selected.get(i); + assertTrue(selected.advanceExact(i)); + final double actual = selected.doubleValue(); double expected = 0.0; - values.setDocument(i); - int numValues = values.count(); - if (numValues == 0) { + if (values.advanceExact(i) == false) { expected = missingValue; } else { + int numValues = values.docValueCount(); if (mode == MultiValueMode.MAX) { expected = Long.MIN_VALUE; } else if (mode == MultiValueMode.MIN) { @@ -297,11 +329,11 @@ private void verify(SortedNumericDoubleValues values, int maxDoc) { } for (int j = 0; j < numValues; ++j) { if (mode == MultiValueMode.SUM || mode == MultiValueMode.AVG) { - expected += values.valueAt(j); + expected += values.nextValue(); } else if (mode == MultiValueMode.MIN) { - expected = Math.min(expected, values.valueAt(j)); + expected = Math.min(expected, values.nextValue()); } else if (mode == MultiValueMode.MAX) { - expected = Math.max(expected, values.valueAt(j)); + expected = Math.max(expected, values.nextValue()); } } if (mode == MultiValueMode.AVG) { @@ -309,9 +341,15 @@ private void verify(SortedNumericDoubleValues values, int maxDoc) { } else if (mode == MultiValueMode.MEDIAN) { int value = numValues/2; if (numValues % 2 == 0) { - expected = (values.valueAt(value - 1) + values.valueAt(value))/2.0; + for (int j = 0; j < value - 1; ++j) { + values.nextValue(); + } + expected = (values.nextValue() + values.nextValue())/2.0; } else { - expected = values.valueAt(value); + for (int j = 0; j < value; ++j) { + values.nextValue(); + } + expected = values.nextValue(); } } } @@ -328,7 +366,8 @@ private void verify(SortedNumericDoubleValues values, int maxDoc, FixedBitSet ro final NumericDoubleValues selected = mode.select(values, missingValue, rootDocs, new BitSetIterator(innerDocs, 0L), maxDoc); int prevRoot = -1; for (int root = rootDocs.nextSetBit(0); root != -1; root = root + 1 < maxDoc ? rootDocs.nextSetBit(root + 1) : -1) { - final double actual = selected.get(root); + assertTrue(selected.advanceExact(root)); + final double actual = selected.doubleValue();; double expected = 0.0; if (mode == MultiValueMode.MAX) { expected = Long.MIN_VALUE; @@ -337,16 +376,17 @@ private void verify(SortedNumericDoubleValues values, int maxDoc, FixedBitSet ro } int numValues = 0; for (int child = innerDocs.nextSetBit(prevRoot + 1); child != -1 && child < root; child = innerDocs.nextSetBit(child + 1)) { - values.setDocument(child); - for (int j = 0; j < values.count(); ++j) { - if (mode == MultiValueMode.SUM || mode == MultiValueMode.AVG) { - expected += values.valueAt(j); - } else if (mode == MultiValueMode.MIN) { - expected = Math.min(expected, values.valueAt(j)); - } else if (mode == MultiValueMode.MAX) { - expected = Math.max(expected, values.valueAt(j)); + if (values.advanceExact(child)) { + for (int j = 0; j < values.docValueCount(); ++j) { + if (mode == MultiValueMode.SUM || mode == MultiValueMode.AVG) { + expected += values.nextValue(); + } else if (mode == MultiValueMode.MIN) { + expected = Math.min(expected, values.nextValue()); + } else if (mode == MultiValueMode.MAX) { + expected = Math.max(expected, values.nextValue()); + } + ++numValues; } - ++numValues; } } if (numValues == 0) { @@ -380,13 +420,19 @@ public void testSingleValuedStrings() throws Exception { } } } - final BinaryDocValues singleValues = new BinaryDocValues() { + final BinaryDocValues singleValues = new AbstractBinaryDocValues() { + int docID; @Override - public BytesRef get(int docID) { + public boolean advanceExact(int target) throws IOException { + docID = target; + return docsWithValue == null || docsWithValue.get(docID); + } + @Override + public BytesRef binaryValue() { return BytesRef.deepCopyOf(array[docID]); } }; - final SortedBinaryDocValues multiValues = FieldData.singleton(singleValues, docsWithValue); + final SortedBinaryDocValues multiValues = FieldData.singleton(singleValues); verify(multiValues, numDocs); final FixedBitSet rootDocs = randomRootDocs(numDocs); final FixedBitSet innerDocs = randomInnerDocs(rootDocs); @@ -406,19 +452,21 @@ public void testMultiValuedStrings() throws Exception { } final SortedBinaryDocValues multiValues = new SortedBinaryDocValues() { int doc; + int i; @Override - public BytesRef valueAt(int index) { - return BytesRef.deepCopyOf(array[doc][index]); + public BytesRef nextValue() { + return BytesRef.deepCopyOf(array[doc][i++]); } @Override - public void setDocument(int doc) { + public boolean advanceExact(int doc) { this.doc = doc; + return array[doc].length > 0; } @Override - public int count() { + public int docValueCount() { return array[doc].length; } }; @@ -428,26 +476,27 @@ public int count() { verify(multiValues, numDocs, rootDocs, innerDocs); } - private void verify(SortedBinaryDocValues values, int maxDoc) { + private void verify(SortedBinaryDocValues values, int maxDoc) throws IOException { for (BytesRef missingValue : new BytesRef[] { new BytesRef(), new BytesRef(RandomStrings.randomAsciiOfLength(random(), 8)) }) { for (MultiValueMode mode : new MultiValueMode[] {MultiValueMode.MIN, MultiValueMode.MAX}) { final BinaryDocValues selected = mode.select(values, missingValue); for (int i = 0; i < maxDoc; ++i) { - final BytesRef actual = selected.get(i); + assertTrue(selected.advanceExact(i)); + final BytesRef actual = selected.binaryValue(); BytesRef expected = null; - values.setDocument(i); - int numValues = values.count(); - if (numValues == 0) { + if (values.advanceExact(i) == false) { expected = missingValue; } else { + int numValues = values.docValueCount(); for (int j = 0; j < numValues; ++j) { if (expected == null) { - expected = BytesRef.deepCopyOf(values.valueAt(j)); + expected = BytesRef.deepCopyOf(values.nextValue()); } else { + BytesRef value = values.nextValue(); if (mode == MultiValueMode.MIN) { - expected = expected.compareTo(values.valueAt(j)) <= 0 ? expected : BytesRef.deepCopyOf(values.valueAt(j)); + expected = expected.compareTo(value) <= 0 ? expected : BytesRef.deepCopyOf(value); } else if (mode == MultiValueMode.MAX) { - expected = expected.compareTo(values.valueAt(j)) > 0 ? expected : BytesRef.deepCopyOf(values.valueAt(j)); + expected = expected.compareTo(value) > 0 ? expected : BytesRef.deepCopyOf(value); } } } @@ -468,18 +517,21 @@ private void verify(SortedBinaryDocValues values, int maxDoc, FixedBitSet rootDo final BinaryDocValues selected = mode.select(values, missingValue, rootDocs, new BitSetIterator(innerDocs, 0L), maxDoc); int prevRoot = -1; for (int root = rootDocs.nextSetBit(0); root != -1; root = root + 1 < maxDoc ? rootDocs.nextSetBit(root + 1) : -1) { - final BytesRef actual = selected.get(root); + assertTrue(selected.advanceExact(root)); + final BytesRef actual = selected.binaryValue(); BytesRef expected = null; for (int child = innerDocs.nextSetBit(prevRoot + 1); child != -1 && child < root; child = innerDocs.nextSetBit(child + 1)) { - values.setDocument(child); - for (int j = 0; j < values.count(); ++j) { - if (expected == null) { - expected = BytesRef.deepCopyOf(values.valueAt(j)); - } else { - if (mode == MultiValueMode.MIN) { - expected = expected.compareTo(values.valueAt(j)) <= 0 ? expected : BytesRef.deepCopyOf(values.valueAt(j)); - } else if (mode == MultiValueMode.MAX) { - expected = expected.compareTo(values.valueAt(j)) > 0 ? expected : BytesRef.deepCopyOf(values.valueAt(j)); + if (values.advanceExact(child)) { + for (int j = 0; j < values.docValueCount(); ++j) { + if (expected == null) { + expected = BytesRef.deepCopyOf(values.nextValue()); + } else { + BytesRef value = values.nextValue(); + if (mode == MultiValueMode.MIN) { + expected = expected.compareTo(value) <= 0 ? expected : BytesRef.deepCopyOf(value); + } else if (mode == MultiValueMode.MAX) { + expected = expected.compareTo(value) > 0 ? expected : BytesRef.deepCopyOf(value); + } } } } @@ -507,9 +559,15 @@ public void testSingleValuedOrds() throws Exception { array[i] = -1; } } - final SortedDocValues singleValues = new SortedDocValues() { + final SortedDocValues singleValues = new AbstractSortedDocValues() { + private int docID; @Override - public int getOrd(int docID) { + public boolean advanceExact(int target) throws IOException { + docID = target; + return array[docID] != -1; + } + @Override + public int ordValue() { return array[docID]; } @@ -523,7 +581,7 @@ public int getValueCount() { return 1 << 20; } }; - final RandomAccessOrds multiValues = (RandomAccessOrds) DocValues.singleton(singleValues); + final SortedSetDocValues multiValues = DocValues.singleton(singleValues); verify(multiValues, numDocs); final FixedBitSet rootDocs = randomRootDocs(numDocs); final FixedBitSet innerDocs = randomInnerDocs(rootDocs); @@ -540,27 +598,23 @@ public void testMultiValuedOrds() throws Exception { } array[i] = values; } - final RandomAccessOrds multiValues = new RandomAccessOrds() { + final SortedSetDocValues multiValues = new AbstractSortedSetDocValues() { int doc; - - @Override - public long ordAt(int index) { - return array[doc][index]; - } - - @Override - public int cardinality() { - return array[doc].length; - } + int i; @Override public long nextOrd() { - throw new UnsupportedOperationException(); + if (i < array[doc].length) { + return array[doc][i++]; + } else { + return NO_MORE_ORDS; + } } @Override - public void setDocument(int docID) { + public boolean advanceExact(int docID) { this.doc = docID; + return array[doc].length > 0; } @Override @@ -579,21 +633,22 @@ public long getValueCount() { verify(multiValues, numDocs, rootDocs, innerDocs); } - private void verify(RandomAccessOrds values, int maxDoc) { + private void verify(SortedSetDocValues values, int maxDoc) throws IOException { for (MultiValueMode mode : new MultiValueMode[] {MultiValueMode.MIN, MultiValueMode.MAX}) { final SortedDocValues selected = mode.select(values); for (int i = 0; i < maxDoc; ++i) { - final long actual = selected.getOrd(i); + final long actual = selected.advanceExact(i) ? selected.ordValue() : -1; int expected = -1; - values.setDocument(i); - for (int j = 0; j < values.cardinality(); ++j) { - if (expected == -1) { - expected = (int) values.ordAt(j); - } else { - if (mode == MultiValueMode.MIN) { - expected = Math.min(expected, (int)values.ordAt(j)); - } else if (mode == MultiValueMode.MAX) { - expected = Math.max(expected, (int)values.ordAt(j)); + if (values.advanceExact(i)) { + for (long ord = values.nextOrd(); ord != SortedSetDocValues.NO_MORE_ORDS; ord = values.nextOrd()) { + if (expected == -1) { + expected = (int) ord; + } else { + if (mode == MultiValueMode.MIN) { + expected = Math.min(expected, (int) ord); + } else if (mode == MultiValueMode.MAX) { + expected = Math.max(expected, (int) ord); + } } } } @@ -603,23 +658,24 @@ private void verify(RandomAccessOrds values, int maxDoc) { } } - private void verify(RandomAccessOrds values, int maxDoc, FixedBitSet rootDocs, FixedBitSet innerDocs) throws IOException { + private void verify(SortedSetDocValues values, int maxDoc, FixedBitSet rootDocs, FixedBitSet innerDocs) throws IOException { for (MultiValueMode mode : new MultiValueMode[] {MultiValueMode.MIN, MultiValueMode.MAX}) { final SortedDocValues selected = mode.select(values, rootDocs, new BitSetIterator(innerDocs, 0L)); int prevRoot = -1; for (int root = rootDocs.nextSetBit(0); root != -1; root = root + 1 < maxDoc ? rootDocs.nextSetBit(root + 1) : -1) { - final int actual = selected.getOrd(root); + final int actual = selected.advanceExact(root) ? selected.ordValue() : -1; int expected = -1; for (int child = innerDocs.nextSetBit(prevRoot + 1); child != -1 && child < root; child = innerDocs.nextSetBit(child + 1)) { - values.setDocument(child); - for (int j = 0; j < values.cardinality(); ++j) { - if (expected == -1) { - expected = (int) values.ordAt(j); - } else { - if (mode == MultiValueMode.MIN) { - expected = Math.min(expected, (int)values.ordAt(j)); - } else if (mode == MultiValueMode.MAX) { - expected = Math.max(expected, (int)values.ordAt(j)); + if (values.advanceExact(child)) { + for (long ord = values.nextOrd(); ord != SortedSetDocValues.NO_MORE_ORDS; ord = values.nextOrd()) { + if (expected == -1) { + expected = (int) ord; + } else { + if (mode == MultiValueMode.MIN) { + expected = Math.min(expected, (int) ord); + } else if (mode == MultiValueMode.MAX) { + expected = Math.max(expected, (int) ord); + } } } } @@ -647,27 +703,33 @@ public void testUnsortedSingleValuedDoubles() throws Exception { } } final NumericDoubleValues singleValues = new NumericDoubleValues() { + private int docID; @Override - public double get(int docID) { + public boolean advanceExact(int doc) throws IOException { + docID = doc; + return docsWithValue == null || docsWithValue.get(docID); + } + @Override + public double doubleValue() { return array[docID]; } }; - final SortedNumericDoubleValues singletonValues = FieldData.singleton(singleValues, docsWithValue); + final SortedNumericDoubleValues singletonValues = FieldData.singleton(singleValues); final MultiValueMode.UnsortedNumericDoubleValues multiValues = new MultiValueMode.UnsortedNumericDoubleValues() { @Override - public int count() { - return singletonValues.count(); + public int docValueCount() { + return singletonValues.docValueCount(); } @Override - public void setDocument(int doc) { - singletonValues.setDocument(doc); + public boolean advanceExact(int doc) throws IOException { + return singletonValues.advanceExact(doc); } @Override - public double valueAt(int index) { - return Math.cos(singletonValues.valueAt(index)); + public double nextValue() throws IOException { + return Math.cos(singletonValues.nextValue()); } }; verify(multiValues, numDocs); @@ -686,37 +748,40 @@ public void testUnsortedMultiValuedDoubles() throws Exception { } final MultiValueMode.UnsortedNumericDoubleValues multiValues = new MultiValueMode.UnsortedNumericDoubleValues() { int doc; + int i; @Override - public int count() { + public int docValueCount() { return array[doc].length; } @Override - public void setDocument(int doc) { + public boolean advanceExact(int doc) { this.doc = doc; + i = 0; + return array[doc].length > 0; } @Override - public double valueAt(int index) { - return Math.sin(array[doc][index]); + public double nextValue() { + return Math.sin(array[doc][i++]); } }; verify(multiValues, numDocs); } - private void verify(MultiValueMode.UnsortedNumericDoubleValues values, int maxDoc) { + private void verify(MultiValueMode.UnsortedNumericDoubleValues values, int maxDoc) throws IOException { for (double missingValue : new double[] { 0, randomDouble() }) { for (MultiValueMode mode : new MultiValueMode[] {MultiValueMode.MIN, MultiValueMode.MAX, MultiValueMode.SUM, MultiValueMode.AVG}) { final NumericDoubleValues selected = mode.select(values, missingValue); for (int i = 0; i < maxDoc; ++i) { - final double actual = selected.get(i); + assertTrue(selected.advanceExact(i)); + final double actual = selected.doubleValue(); double expected = 0.0; - values.setDocument(i); - int numValues = values.count(); - if (numValues == 0) { + if (values.advanceExact(i) == false) { expected = missingValue; } else { + int numValues = values.docValueCount(); if (mode == MultiValueMode.MAX) { expected = Long.MIN_VALUE; } else if (mode == MultiValueMode.MIN) { @@ -724,11 +789,11 @@ private void verify(MultiValueMode.UnsortedNumericDoubleValues values, int maxDo } for (int j = 0; j < numValues; ++j) { if (mode == MultiValueMode.SUM || mode == MultiValueMode.AVG) { - expected += values.valueAt(j); + expected += values.nextValue(); } else if (mode == MultiValueMode.MIN) { - expected = Math.min(expected, values.valueAt(j)); + expected = Math.min(expected, values.nextValue()); } else if (mode == MultiValueMode.MAX) { - expected = Math.max(expected, values.valueAt(j)); + expected = Math.max(expected, values.nextValue()); } } if (mode == MultiValueMode.AVG) { diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java b/core/src/test/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java index a0f7bbfcba43b..25be83a7ca5f8 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java @@ -225,7 +225,7 @@ protected A searchAndReduc List aggs = new ArrayList<> (); Query rewritten = searcher.rewrite(query); - Weight weight = searcher.createWeight(rewritten, true); + Weight weight = searcher.createWeight(rewritten, true, 1f); C root = createAggregator(builder, searcher, fieldTypes); try { for (ShardSearcher subSearcher : subSearchers) { diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/range/BinaryRangeAggregatorTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/range/BinaryRangeAggregatorTests.java index 6ba6da3b5552f..6ed2c1a3a8c9e 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/range/BinaryRangeAggregatorTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/range/BinaryRangeAggregatorTests.java @@ -23,9 +23,9 @@ import java.util.HashSet; import java.util.Set; -import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.TestUtil; +import org.elasticsearch.index.fielddata.AbstractSortedSetDocValues; import org.elasticsearch.index.fielddata.SortedBinaryDocValues; import org.elasticsearch.search.aggregations.LeafBucketCollector; import org.elasticsearch.search.aggregations.bucket.range.BinaryRangeAggregator.SortedBinaryRangeLeafCollector; @@ -36,7 +36,7 @@ public class BinaryRangeAggregatorTests extends ESTestCase { - private static class FakeSortedSetDocValues extends SortedSetDocValues { + private static class FakeSortedSetDocValues extends AbstractSortedSetDocValues { private final BytesRef[] terms; long[] ords; @@ -47,8 +47,9 @@ private static class FakeSortedSetDocValues extends SortedSetDocValues { } @Override - public void setDocument(int docID) { + public boolean advanceExact(int docID) { i = 0; + return true; } @Override @@ -145,6 +146,7 @@ public void testSortedSetRangeLeafCollectorMultiValued() throws Exception { private static class FakeSortedBinaryDocValues extends SortedBinaryDocValues { private final BytesRef[] terms; + int i; long[] ords; FakeSortedBinaryDocValues(BytesRef[] terms) { @@ -152,18 +154,19 @@ private static class FakeSortedBinaryDocValues extends SortedBinaryDocValues { } @Override - public void setDocument(int docID) { - // no-op + public boolean advanceExact(int docID) { + i = 0; + return true; } @Override - public int count() { + public int docValueCount() { return ords.length; } @Override - public BytesRef valueAt(int index) { - return terms[(int) ords[index]]; + public BytesRef nextValue() { + return terms[(int) ords[i++]]; } } diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/support/ScriptValuesTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/support/ScriptValuesTests.java index f338e961312bd..bcb6d5318b75d 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/support/ScriptValuesTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/support/ScriptValuesTests.java @@ -101,9 +101,9 @@ public void testLongs() { ScriptLongValues scriptValues = new ScriptLongValues(script); for (int i = 0; i < values.length; ++i) { scriptValues.setDocument(i); - assertEquals(values[i].length, scriptValues.count()); + assertEquals(values[i].length, scriptValues.docValueCount()); for (int j = 0; j < values[i].length; ++j) { - assertEquals(values[i][j], scriptValues.valueAt(j)); + assertEquals(values[i][j], scriptValues.nextValue()); } } } @@ -122,9 +122,9 @@ public void testBooleans() { ScriptLongValues scriptValues = new ScriptLongValues(script); for (int i = 0; i < values.length; ++i) { scriptValues.setDocument(i); - assertEquals(values[i].length, scriptValues.count()); + assertEquals(values[i].length, scriptValues.docValueCount()); for (int j = 0; j < values[i].length; ++j) { - assertEquals(values[i][j], scriptValues.valueAt(j) == 1L); + assertEquals(values[i][j], scriptValues.nextValue() == 1L); } } } @@ -143,9 +143,9 @@ public void testDoubles() { ScriptDoubleValues scriptValues = new ScriptDoubleValues(script); for (int i = 0; i < values.length; ++i) { scriptValues.setDocument(i); - assertEquals(values[i].length, scriptValues.count()); + assertEquals(values[i].length, scriptValues.docValueCount()); for (int j = 0; j < values[i].length; ++j) { - assertEquals(values[i][j], scriptValues.valueAt(j)); + assertEquals(values[i][j], scriptValues.nextValue()); } } } @@ -164,9 +164,9 @@ public void testBytes() { ScriptBytesValues scriptValues = new ScriptBytesValues(script); for (int i = 0; i < values.length; ++i) { scriptValues.setDocument(i); - assertEquals(values[i].length, scriptValues.count()); + assertEquals(values[i].length, scriptValues.docValueCount()); for (int j = 0; j < values[i].length; ++j) { - assertEquals(new BytesRef(values[i][j]), scriptValues.valueAt(j)); + assertEquals(new BytesRef(values[i][j]), scriptValues.nextValue()); } } } diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfigTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfigTests.java index 5f8c658f234a9..d0c583257ae87 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfigTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfigTests.java @@ -48,9 +48,9 @@ public void testKeyword() throws Exception { ValuesSource.Bytes valuesSource = config.toValuesSource(context); LeafReaderContext ctx = searcher.reader().leaves().get(0); SortedBinaryDocValues values = valuesSource.bytesValues(ctx); - values.setDocument(0); - assertEquals(1, values.count()); - assertEquals(new BytesRef("abc"), values.valueAt(0)); + assertTrue(values.advanceExact(0)); + assertEquals(1, values.docValueCount()); + assertEquals(new BytesRef("abc"), values.nextValue()); } } @@ -70,16 +70,15 @@ public void testEmptyKeyword() throws Exception { ValuesSource.Bytes valuesSource = config.toValuesSource(context); LeafReaderContext ctx = searcher.reader().leaves().get(0); SortedBinaryDocValues values = valuesSource.bytesValues(ctx); - values.setDocument(0); - assertEquals(0, values.count()); + assertFalse(values.advanceExact(0)); config = ValuesSourceConfig.resolve( context, null, "bytes", null, "abc", null, null); valuesSource = config.toValuesSource(context); values = valuesSource.bytesValues(ctx); - values.setDocument(0); - assertEquals(1, values.count()); - assertEquals(new BytesRef("abc"), values.valueAt(0)); + assertTrue(values.advanceExact(0)); + assertEquals(1, values.docValueCount()); + assertEquals(new BytesRef("abc"), values.nextValue()); } } @@ -102,9 +101,9 @@ public void testUnmappedKeyword() throws Exception { valuesSource = config.toValuesSource(context); LeafReaderContext ctx = searcher.reader().leaves().get(0); SortedBinaryDocValues values = valuesSource.bytesValues(ctx); - values.setDocument(0); - assertEquals(1, values.count()); - assertEquals(new BytesRef("abc"), values.valueAt(0)); + assertTrue(values.advanceExact(0)); + assertEquals(1, values.docValueCount()); + assertEquals(new BytesRef("abc"), values.nextValue()); } } @@ -124,9 +123,9 @@ public void testLong() throws Exception { ValuesSource.Numeric valuesSource = config.toValuesSource(context); LeafReaderContext ctx = searcher.reader().leaves().get(0); SortedNumericDocValues values = valuesSource.longValues(ctx); - values.setDocument(0); - assertEquals(1, values.count()); - assertEquals(42, values.valueAt(0)); + assertTrue(values.advanceExact(0)); + assertEquals(1, values.docValueCount()); + assertEquals(42, values.nextValue()); } } @@ -146,16 +145,15 @@ public void testEmptyLong() throws Exception { ValuesSource.Numeric valuesSource = config.toValuesSource(context); LeafReaderContext ctx = searcher.reader().leaves().get(0); SortedNumericDocValues values = valuesSource.longValues(ctx); - values.setDocument(0); - assertEquals(0, values.count()); + assertFalse(values.advanceExact(0)); config = ValuesSourceConfig.resolve( context, null, "long", null, 42, null, null); valuesSource = config.toValuesSource(context); values = valuesSource.longValues(ctx); - values.setDocument(0); - assertEquals(1, values.count()); - assertEquals(42, values.valueAt(0)); + assertTrue(values.advanceExact(0)); + assertEquals(1, values.docValueCount()); + assertEquals(42, values.nextValue()); } } @@ -179,9 +177,9 @@ public void testUnmappedLong() throws Exception { valuesSource = config.toValuesSource(context); LeafReaderContext ctx = searcher.reader().leaves().get(0); SortedNumericDocValues values = valuesSource.longValues(ctx); - values.setDocument(0); - assertEquals(1, values.count()); - assertEquals(42, values.valueAt(0)); + assertTrue(values.advanceExact(0)); + assertEquals(1, values.docValueCount()); + assertEquals(42, values.nextValue()); } } @@ -201,9 +199,9 @@ public void testBoolean() throws Exception { ValuesSource.Numeric valuesSource = config.toValuesSource(context); LeafReaderContext ctx = searcher.reader().leaves().get(0); SortedNumericDocValues values = valuesSource.longValues(ctx); - values.setDocument(0); - assertEquals(1, values.count()); - assertEquals(1, values.valueAt(0)); + assertTrue(values.advanceExact(0)); + assertEquals(1, values.docValueCount()); + assertEquals(1, values.nextValue()); } } @@ -223,16 +221,15 @@ public void testEmptyBoolean() throws Exception { ValuesSource.Numeric valuesSource = config.toValuesSource(context); LeafReaderContext ctx = searcher.reader().leaves().get(0); SortedNumericDocValues values = valuesSource.longValues(ctx); - values.setDocument(0); - assertEquals(0, values.count()); + assertFalse(values.advanceExact(0)); config = ValuesSourceConfig.resolve( context, null, "bool", null, true, null, null); valuesSource = config.toValuesSource(context); values = valuesSource.longValues(ctx); - values.setDocument(0); - assertEquals(1, values.count()); - assertEquals(1, values.valueAt(0)); + assertTrue(values.advanceExact(0)); + assertEquals(1, values.docValueCount()); + assertEquals(1, values.nextValue()); } } @@ -256,9 +253,9 @@ public void testUnmappedBoolean() throws Exception { valuesSource = config.toValuesSource(context); LeafReaderContext ctx = searcher.reader().leaves().get(0); SortedNumericDocValues values = valuesSource.longValues(ctx); - values.setDocument(0); - assertEquals(1, values.count()); - assertEquals(1, values.valueAt(0)); + assertTrue(values.advanceExact(0)); + assertEquals(1, values.docValueCount()); + assertEquals(1, values.nextValue()); } } } diff --git a/core/src/test/java/org/elasticsearch/search/basic/SearchWithRandomExceptionsIT.java b/core/src/test/java/org/elasticsearch/search/basic/SearchWithRandomExceptionsIT.java index 067064dfc156f..5b68d3cd1f1f6 100644 --- a/core/src/test/java/org/elasticsearch/search/basic/SearchWithRandomExceptionsIT.java +++ b/core/src/test/java/org/elasticsearch/search/basic/SearchWithRandomExceptionsIT.java @@ -243,6 +243,11 @@ public RandomExceptionDirectoryReaderWrapper(DirectoryReader in, Settings settin protected DirectoryReader doWrapDirectoryReader(DirectoryReader in) throws IOException { return new RandomExceptionDirectoryReaderWrapper(in, settings); } + + @Override + public CacheHelper getReaderCacheHelper() { + return in.getReaderCacheHelper(); + } } diff --git a/core/src/test/java/org/elasticsearch/search/fetch/subphase/NestedChildrenFilterTests.java b/core/src/test/java/org/elasticsearch/search/fetch/subphase/NestedChildrenFilterTests.java index c4c180ab8580d..7c85841d54955 100644 --- a/core/src/test/java/org/elasticsearch/search/fetch/subphase/NestedChildrenFilterTests.java +++ b/core/src/test/java/org/elasticsearch/search/fetch/subphase/NestedChildrenFilterTests.java @@ -21,7 +21,7 @@ import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; -import org.apache.lucene.document.LegacyIntField; +import org.apache.lucene.document.StoredField; import org.apache.lucene.document.StringField; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.LeafReaderContext; @@ -64,7 +64,7 @@ public void testNestedChildrenFilter() throws Exception { Document parenDoc = new Document(); parenDoc.add(new StringField("type", "parent", Field.Store.NO)); - parenDoc.add(new LegacyIntField("num_child_docs", numChildDocs, Field.Store.YES)); + parenDoc.add(new StoredField("num_child_docs", numChildDocs)); docs.add(parenDoc); writer.addDocuments(docs); } diff --git a/core/src/test/java/org/elasticsearch/search/functionscore/QueryRescorerIT.java b/core/src/test/java/org/elasticsearch/search/functionscore/QueryRescorerIT.java index fc9dfb8844967..18db8cd539e0d 100644 --- a/core/src/test/java/org/elasticsearch/search/functionscore/QueryRescorerIT.java +++ b/core/src/test/java/org/elasticsearch/search/functionscore/QueryRescorerIT.java @@ -587,12 +587,12 @@ public void testScoring() throws Exception { String[] intToEnglish = new String[] { English.intToEnglish(i), English.intToEnglish(i + 1), English.intToEnglish(i + 2), English.intToEnglish(i + 3) }; - QueryBuilder query = boolQuery().disableCoord(true) + QueryBuilder query = boolQuery() .should(functionScoreQuery(termQuery("field1", intToEnglish[0]), weightFactorFunction(2.0f)).boostMode(REPLACE)) .should(functionScoreQuery(termQuery("field1", intToEnglish[1]), weightFactorFunction(3.0f)).boostMode(REPLACE)) .should(functionScoreQuery(termQuery("field1", intToEnglish[2]), weightFactorFunction(5.0f)).boostMode(REPLACE)) .should(functionScoreQuery(termQuery("field1", intToEnglish[3]), weightFactorFunction(0.2f)).boostMode(REPLACE)); - QueryRescorerBuilder rescoreQuery = queryRescorer(boolQuery().disableCoord(true) + QueryRescorerBuilder rescoreQuery = queryRescorer(boolQuery() .should(functionScoreQuery(termQuery("field1", intToEnglish[0]), weightFactorFunction(5.0f)).boostMode(REPLACE)) .should(functionScoreQuery(termQuery("field1", intToEnglish[1]), weightFactorFunction(7.0f)).boostMode(REPLACE)) .should(functionScoreQuery(termQuery("field1", intToEnglish[3]), weightFactorFunction(0.0f)).boostMode(REPLACE))); From c199391f217a3d75dc823e75ffc8683b58cb0cd4 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Thu, 6 Apr 2017 14:31:52 +0200 Subject: [PATCH 28/87] Fix grouping compile errors. --- .../grouping/CollapsingDocValuesSource.java | 78 ++++++++++++------- .../grouping/CollapsingTopDocsCollector.java | 6 +- 2 files changed, 51 insertions(+), 33 deletions(-) diff --git a/core/src/main/java/org/apache/lucene/search/grouping/CollapsingDocValuesSource.java b/core/src/main/java/org/apache/lucene/search/grouping/CollapsingDocValuesSource.java index 5bc8afb347c71..ad3ee51bc3494 100644 --- a/core/src/main/java/org/apache/lucene/search/grouping/CollapsingDocValuesSource.java +++ b/core/src/main/java/org/apache/lucene/search/grouping/CollapsingDocValuesSource.java @@ -27,8 +27,9 @@ import org.apache.lucene.index.SortedNumericDocValues; import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.util.ArrayUtil; -import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.index.fielddata.AbstractNumericDocValues; +import org.elasticsearch.index.fielddata.AbstractSortedDocValues; import java.io.IOException; @@ -42,7 +43,7 @@ abstract class CollapsingDocValuesSource { this.field = field; } - abstract T get(int doc); + abstract T get(int doc) throws IOException; abstract T copy(T value, T reuse); @@ -54,16 +55,15 @@ abstract class CollapsingDocValuesSource { */ static class Numeric extends CollapsingDocValuesSource { private NumericDocValues values; - private Bits docsWithField; Numeric(String field) throws IOException { super(field); } @Override - public Long get(int doc) { - if (docsWithField.get(doc)) { - return values.get(doc); + public Long get(int doc) throws IOException { + if (values.advanceExact(doc)) { + return values.longValue(); } else { return null; } @@ -79,10 +79,8 @@ public void setNextReader(LeafReader reader) throws IOException { DocValuesType type = getDocValuesType(reader, field); if (type == null || type == DocValuesType.NONE) { values = DocValues.emptyNumeric(); - docsWithField = new Bits.MatchNoBits(reader.maxDoc()); return ; } - docsWithField = DocValues.getDocsWithField(reader, field); switch (type) { case NUMERIC: values = DocValues.getNumeric(reader, field); @@ -92,17 +90,29 @@ public void setNextReader(LeafReader reader) throws IOException { final SortedNumericDocValues sorted = DocValues.getSortedNumeric(reader, field); values = DocValues.unwrapSingleton(sorted); if (values == null) { - values = new NumericDocValues() { + values = new AbstractNumericDocValues() { + + private long value; + @Override - public long get(int docID) { - sorted.setDocument(docID); - assert sorted.count() > 0; - if (sorted.count() > 1) { - throw new IllegalStateException("failed to collapse " + docID + - ", the collapse field must be single valued"); + public boolean advanceExact(int target) throws IOException { + if (sorted.advanceExact(target)) { + if (sorted.docValueCount() > 1) { + throw new IllegalStateException("failed to collapse " + target + + ", the collapse field must be single valued"); + } + value = sorted.nextValue(); + return true; + } else { + return false; } - return sorted.valueAt(0); } + + @Override + public long longValue() throws IOException { + return value; + } + }; } break; @@ -119,7 +129,6 @@ public long get(int docID) { * Fails with an {@link IllegalStateException} if a document contains multiple values for the specified field. */ static class Keyword extends CollapsingDocValuesSource { - private Bits docsWithField; private SortedDocValues values; Keyword(String field) throws IOException { @@ -127,9 +136,9 @@ static class Keyword extends CollapsingDocValuesSource { } @Override - public BytesRef get(int doc) { - if (docsWithField.get(doc)) { - return values.get(doc); + public BytesRef get(int doc) throws IOException { + if (values.advanceExact(doc)) { + return values.binaryValue(); } else { return null; } @@ -156,10 +165,8 @@ public void setNextReader(LeafReader reader) throws IOException { DocValuesType type = getDocValuesType(reader, field); if (type == null || type == DocValuesType.NONE) { values = DocValues.emptySorted(); - docsWithField = new Bits.MatchNoBits(reader.maxDoc()); return ; } - docsWithField = DocValues.getDocsWithField(reader, field); switch (type) { case SORTED: values = DocValues.getSorted(reader, field); @@ -169,20 +176,31 @@ public void setNextReader(LeafReader reader) throws IOException { final SortedSetDocValues sorted = DocValues.getSortedSet(reader, field); values = DocValues.unwrapSingleton(sorted); if (values == null) { - values = new SortedDocValues() { + values = new AbstractSortedDocValues() { + + private int ord; + @Override - public int getOrd(int docID) { - sorted.setDocument(docID); - int ord = (int) sorted.nextOrd(); - if (sorted.nextOrd() != SortedSetDocValues.NO_MORE_ORDS) { - throw new IllegalStateException("failed to collapse " + docID + - ", the collapse field must be single valued"); + public boolean advanceExact(int target) throws IOException { + if (sorted.advanceExact(target)) { + ord = (int) sorted.nextOrd(); + if (sorted.nextOrd() != SortedSetDocValues.NO_MORE_ORDS) { + throw new IllegalStateException("failed to collapse " + target + + ", the collapse field must be single valued"); + } + return true; + } else { + return false; } + } + + @Override + public int ordValue() { return ord; } @Override - public BytesRef lookupOrd(int ord) { + public BytesRef lookupOrd(int ord) throws IOException { return sorted.lookupOrd(ord); } diff --git a/core/src/main/java/org/apache/lucene/search/grouping/CollapsingTopDocsCollector.java b/core/src/main/java/org/apache/lucene/search/grouping/CollapsingTopDocsCollector.java index 955a63e5483b7..33d1b94685a90 100644 --- a/core/src/main/java/org/apache/lucene/search/grouping/CollapsingTopDocsCollector.java +++ b/core/src/main/java/org/apache/lucene/search/grouping/CollapsingTopDocsCollector.java @@ -65,7 +65,7 @@ private CollapsingTopDocsCollector(String collapseField, Sort sort, * {@link CollapseTopFieldDocs}. The collapsing needs only one pass so we can create the final top docs at the end * of the first pass. */ - public CollapseTopFieldDocs getTopDocs() { + public CollapseTopFieldDocs getTopDocs() throws IOException { Collection> groups = super.getTopGroups(0, true); if (groups == null) { return new CollapseTopFieldDocs(collapseField, totalHitCount, new ScoreDoc[0], @@ -136,7 +136,7 @@ protected void doSetNextReader(LeafReaderContext readerContext) throws IOExcepti } @Override - protected Long getDocGroupValue(int doc) { + protected Long getDocGroupValue(int doc) throws IOException { return source.get(doc); } @@ -162,7 +162,7 @@ protected void doSetNextReader(LeafReaderContext readerContext) throws IOExcepti } @Override - protected BytesRef getDocGroupValue(int doc) { + protected BytesRef getDocGroupValue(int doc) throws IOException { return source.get(doc); } From 438a840279070ba49a8b9ae316529aaf60e17fc4 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Thu, 6 Apr 2017 14:35:11 +0200 Subject: [PATCH 29/87] Fix compile errors in ReplaceMissingTests. --- .../fieldcomparator/ReplaceMissingTests.java | 45 ++++++++++++------- 1 file changed, 30 insertions(+), 15 deletions(-) diff --git a/core/src/test/java/org/elasticsearch/index/fielddata/fieldcomparator/ReplaceMissingTests.java b/core/src/test/java/org/elasticsearch/index/fielddata/fieldcomparator/ReplaceMissingTests.java index a291311c3bc7f..f897fc70861cc 100644 --- a/core/src/test/java/org/elasticsearch/index/fielddata/fieldcomparator/ReplaceMissingTests.java +++ b/core/src/test/java/org/elasticsearch/index/fielddata/fieldcomparator/ReplaceMissingTests.java @@ -62,18 +62,24 @@ public void test() throws Exception { assertEquals("cat", dv.lookupOrd(0).utf8ToString()); assertEquals("dog", dv.lookupOrd(1).utf8ToString()); - assertEquals(0, dv.getOrd(0)); - assertEquals(0, dv.getOrd(1)); - assertEquals(1, dv.getOrd(2)); + assertTrue(dv.advanceExact(0)); + assertEquals(0, dv.ordValue()); + assertTrue(dv.advanceExact(1)); + assertEquals(0, dv.ordValue()); + assertTrue(dv.advanceExact(2)); + assertEquals(1, dv.ordValue()); dv = new BytesRefFieldComparatorSource.ReplaceMissing(raw, new BytesRef("dog")); assertEquals(2, dv.getValueCount()); assertEquals("cat", dv.lookupOrd(0).utf8ToString()); assertEquals("dog", dv.lookupOrd(1).utf8ToString()); - assertEquals(0, dv.getOrd(0)); - assertEquals(1, dv.getOrd(1)); - assertEquals(1, dv.getOrd(2)); + assertTrue(dv.advanceExact(0)); + assertEquals(0, dv.ordValue()); + assertTrue(dv.advanceExact(1)); + assertEquals(1, dv.ordValue()); + assertTrue(dv.advanceExact(2)); + assertEquals(1, dv.ordValue()); // non-existing values dv = new BytesRefFieldComparatorSource.ReplaceMissing(raw, new BytesRef("apple")); @@ -82,9 +88,12 @@ public void test() throws Exception { assertEquals("cat", dv.lookupOrd(1).utf8ToString()); assertEquals("dog", dv.lookupOrd(2).utf8ToString()); - assertEquals(1, dv.getOrd(0)); - assertEquals(0, dv.getOrd(1)); - assertEquals(2, dv.getOrd(2)); + assertTrue(dv.advanceExact(0)); + assertEquals(1, dv.ordValue()); + assertTrue(dv.advanceExact(1)); + assertEquals(0, dv.ordValue()); + assertTrue(dv.advanceExact(2)); + assertEquals(2, dv.ordValue()); dv = new BytesRefFieldComparatorSource.ReplaceMissing(raw, new BytesRef("company")); assertEquals(3, dv.getValueCount()); @@ -92,9 +101,12 @@ public void test() throws Exception { assertEquals("company", dv.lookupOrd(1).utf8ToString()); assertEquals("dog", dv.lookupOrd(2).utf8ToString()); - assertEquals(0, dv.getOrd(0)); - assertEquals(1, dv.getOrd(1)); - assertEquals(2, dv.getOrd(2)); + assertTrue(dv.advanceExact(0)); + assertEquals(0, dv.ordValue()); + assertTrue(dv.advanceExact(1)); + assertEquals(1, dv.ordValue()); + assertTrue(dv.advanceExact(2)); + assertEquals(2, dv.ordValue()); dv = new BytesRefFieldComparatorSource.ReplaceMissing(raw, new BytesRef("ebay")); assertEquals(3, dv.getValueCount()); @@ -102,9 +114,12 @@ public void test() throws Exception { assertEquals("dog", dv.lookupOrd(1).utf8ToString()); assertEquals("ebay", dv.lookupOrd(2).utf8ToString()); - assertEquals(0, dv.getOrd(0)); - assertEquals(2, dv.getOrd(1)); - assertEquals(1, dv.getOrd(2)); + assertTrue(dv.advanceExact(0)); + assertEquals(0, dv.ordValue()); + assertTrue(dv.advanceExact(1)); + assertEquals(2, dv.ordValue()); + assertTrue(dv.advanceExact(2)); + assertEquals(1, dv.ordValue()); reader.close(); dir.close(); From acc89c2068fa3bda989cd053b8830f380f614eec Mon Sep 17 00:00:00 2001 From: Colin Goodheart-Smithe Date: Thu, 6 Apr 2017 11:32:40 +0100 Subject: [PATCH 30/87] Fixes most compile errors in fielddata classes Does not fix errors in ScriptDocValues --- .../SortableLongBitsNumericDocValues.java | 13 +- ...ortableLongBitsSortedNumericDocValues.java | 16 +-- ...SortableLongBitsToNumericDoubleValues.java | 11 +- ...leLongBitsToSortedNumericDoubleValues.java | 14 ++- .../BytesRefFieldComparatorSource.java | 26 ++-- .../fielddata/ordinals/MultiOrdinals.java | 58 +++++---- .../index/fielddata/ordinals/Ordinals.java | 6 +- .../fielddata/ordinals/OrdinalsBuilder.java | 51 -------- .../ordinals/SinglePackedOrdinals.java | 23 ++-- .../AbstractAtomicGeoPointFieldData.java | 2 +- .../AbstractAtomicParentChildFieldData.java | 16 ++- .../plain/AbstractIndexGeoPointFieldData.java | 117 ------------------ .../plain/AtomicDoubleFieldData.java | 2 +- .../plain/BinaryDVAtomicFieldData.java | 4 +- .../plain/BytesBinaryDVAtomicFieldData.java | 53 ++++---- .../plain/DocValuesIndexFieldData.java | 1 - .../plain/GeoPointDVAtomicFieldData.java | 36 +++--- .../fielddata/plain/IndexIndexFieldData.java | 17 ++- .../plain/LatLonPointDVAtomicFieldData.java | 40 +++--- .../plain/PagedBytesAtomicFieldData.java | 4 +- .../plain/ParentChildIndexFieldData.java | 27 ++-- .../plain/SortedNumericDVIndexFieldData.java | 46 ++++--- 22 files changed, 247 insertions(+), 336 deletions(-) delete mode 100644 core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractIndexGeoPointFieldData.java diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/SortableLongBitsNumericDocValues.java b/core/src/main/java/org/elasticsearch/index/fielddata/SortableLongBitsNumericDocValues.java index 678b42061331f..af1b273d1c142 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/SortableLongBitsNumericDocValues.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/SortableLongBitsNumericDocValues.java @@ -22,12 +22,14 @@ import org.apache.lucene.index.NumericDocValues; import org.apache.lucene.util.NumericUtils; +import java.io.IOException; + /** * {@link NumericDocValues} instance that wraps a {@link NumericDoubleValues} * and converts the doubles to sortable long bits using * {@link NumericUtils#doubleToSortableLong(double)}. */ -final class SortableLongBitsNumericDocValues extends NumericDocValues { +final class SortableLongBitsNumericDocValues extends AbstractNumericDocValues { private final NumericDoubleValues values; @@ -36,8 +38,13 @@ final class SortableLongBitsNumericDocValues extends NumericDocValues { } @Override - public long get(int docID) { - return NumericUtils.doubleToSortableLong(values.get(docID)); + public long longValue() throws IOException { + return NumericUtils.doubleToSortableLong(values.doubleValue()); + } + + @Override + public boolean advanceExact(int target) throws IOException { + return values.advanceExact(target); } /** Return the wrapped values. */ diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/SortableLongBitsSortedNumericDocValues.java b/core/src/main/java/org/elasticsearch/index/fielddata/SortableLongBitsSortedNumericDocValues.java index 9dbf44f14f285..81d33bc432f40 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/SortableLongBitsSortedNumericDocValues.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/SortableLongBitsSortedNumericDocValues.java @@ -22,12 +22,14 @@ import org.apache.lucene.index.SortedNumericDocValues; import org.apache.lucene.util.NumericUtils; +import java.io.IOException; + /** * {@link SortedNumericDocValues} instance that wraps a {@link SortedNumericDoubleValues} * and converts the doubles to sortable long bits using * {@link NumericUtils#doubleToSortableLong(double)}. */ -final class SortableLongBitsSortedNumericDocValues extends SortedNumericDocValues { +final class SortableLongBitsSortedNumericDocValues extends AbstractSortedNumericDocValues { private final SortedNumericDoubleValues values; @@ -36,18 +38,18 @@ final class SortableLongBitsSortedNumericDocValues extends SortedNumericDocValue } @Override - public void setDocument(int doc) { - values.setDocument(doc); + public boolean advanceExact(int target) throws IOException { + return values.advanceExact(target); } @Override - public long valueAt(int index) { - return NumericUtils.doubleToSortableLong(values.valueAt(index)); + public long nextValue() throws IOException { + return NumericUtils.doubleToSortableLong(values.nextValue()); } @Override - public int count() { - return values.count(); + public int docValueCount() { + return values.docValueCount(); } /** Return the wrapped values. */ diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/SortableLongBitsToNumericDoubleValues.java b/core/src/main/java/org/elasticsearch/index/fielddata/SortableLongBitsToNumericDoubleValues.java index 7c150cf319b0b..94bc7168dca41 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/SortableLongBitsToNumericDoubleValues.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/SortableLongBitsToNumericDoubleValues.java @@ -22,6 +22,8 @@ import org.apache.lucene.index.NumericDocValues; import org.apache.lucene.util.NumericUtils; +import java.io.IOException; + /** * {@link NumericDoubleValues} instance that wraps a {@link NumericDocValues} * and converts the doubles to sortable long bits using @@ -36,8 +38,13 @@ final class SortableLongBitsToNumericDoubleValues extends NumericDoubleValues { } @Override - public double get(int docID) { - return NumericUtils.sortableLongToDouble(values.get(docID)); + public double doubleValue() throws IOException { + return NumericUtils.sortableLongToDouble(values.longValue()); + } + + @Override + public boolean advanceExact(int doc) throws IOException { + return values.advanceExact(doc); } /** Return the wrapped values. */ diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/SortableLongBitsToSortedNumericDoubleValues.java b/core/src/main/java/org/elasticsearch/index/fielddata/SortableLongBitsToSortedNumericDoubleValues.java index 2669463170b47..d78b66114b32a 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/SortableLongBitsToSortedNumericDoubleValues.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/SortableLongBitsToSortedNumericDoubleValues.java @@ -22,6 +22,8 @@ import org.apache.lucene.index.SortedNumericDocValues; import org.apache.lucene.util.NumericUtils; +import java.io.IOException; + /** * {@link SortedNumericDoubleValues} instance that wraps a {@link SortedNumericDocValues} * and converts the doubles to sortable long bits using @@ -36,18 +38,18 @@ final class SortableLongBitsToSortedNumericDoubleValues extends SortedNumericDou } @Override - public void setDocument(int doc) { - values.setDocument(doc); + public boolean advanceExact(int target) throws IOException { + return values.advanceExact(target); } @Override - public double valueAt(int index) { - return NumericUtils.sortableLongToDouble(values.valueAt(index)); + public double nextValue() throws IOException { + return NumericUtils.sortableLongToDouble(values.nextValue()); } @Override - public int count() { - return values.count(); + public int docValueCount() { + return values.docValueCount(); } /** Return the wrapped values. */ diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/BytesRefFieldComparatorSource.java b/core/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/BytesRefFieldComparatorSource.java index 48b6a1127a5e0..d9a486bf7de56 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/BytesRefFieldComparatorSource.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/BytesRefFieldComparatorSource.java @@ -21,15 +21,15 @@ import org.apache.lucene.index.BinaryDocValues; import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.RandomAccessOrds; import org.apache.lucene.index.SortedDocValues; +import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.FieldComparator; import org.apache.lucene.search.Scorer; import org.apache.lucene.search.SortField; import org.apache.lucene.util.BitSet; -import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.index.fielddata.AbstractSortedDocValues; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.IndexOrdinalsFieldData; import org.elasticsearch.index.fielddata.SortedBinaryDocValues; @@ -89,7 +89,7 @@ public FieldComparator newComparator(String fieldname, int numHits, int sortP @Override protected SortedDocValues getSortedDocValues(LeafReaderContext context, String field) throws IOException { - final RandomAccessOrds values = ((IndexOrdinalsFieldData) indexFieldData).load(context).getOrdinalsValues(); + final SortedSetDocValues values = ((IndexOrdinalsFieldData) indexFieldData).load(context).getOrdinalsValues(); final SortedDocValues selectedValues; if (nested == null) { selectedValues = sortMode.select(values); @@ -131,11 +131,6 @@ protected BinaryDocValues getBinaryDocValues(LeafReaderContext context, String f return selectedValues; } - @Override - protected Bits getDocsWithField(LeafReaderContext context, String field) throws IOException { - return new Bits.MatchAllBits(context.reader().maxDoc()); - } - @Override protected boolean isNull(int doc, BytesRef term) { return term == nullPlaceHolder; @@ -154,13 +149,13 @@ public void setScorer(Scorer scorer) { * are replaced with the specified term */ // TODO: move this out if we need it for other reasons - static class ReplaceMissing extends SortedDocValues { + static class ReplaceMissing extends AbstractSortedDocValues { final SortedDocValues in; final int substituteOrd; final BytesRef substituteTerm; final boolean exists; - ReplaceMissing(SortedDocValues in, BytesRef term) { + ReplaceMissing(SortedDocValues in, BytesRef term) throws IOException { this.in = in; this.substituteTerm = term; int sub = in.lookupTerm(term); @@ -174,8 +169,8 @@ static class ReplaceMissing extends SortedDocValues { } @Override - public int getOrd(int docID) { - int ord = in.getOrd(docID); + public int ordValue() { + int ord = in.ordValue(); if (ord < 0) { return substituteOrd; } else if (exists == false && ord >= substituteOrd) { @@ -185,6 +180,11 @@ public int getOrd(int docID) { } } + @Override + public boolean advanceExact(int target) throws IOException { + return in.advanceExact(target); + } + @Override public int getValueCount() { if (exists) { @@ -195,7 +195,7 @@ public int getValueCount() { } @Override - public BytesRef lookupOrd(int ord) { + public BytesRef lookupOrd(int ord) throws IOException { if (ord == substituteOrd) { return substituteTerm; } else if (exists == false && ord > substituteOrd) { diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/MultiOrdinals.java b/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/MultiOrdinals.java index d85073a06882e..cb277faeebb41 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/MultiOrdinals.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/MultiOrdinals.java @@ -20,16 +20,17 @@ package org.elasticsearch.index.fielddata.ordinals; import org.apache.lucene.index.DocValues; -import org.apache.lucene.index.RandomAccessOrds; -import org.apache.lucene.index.SortedDocValues; +import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.util.Accountable; import org.apache.lucene.util.Accountables; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.LongsRef; import org.apache.lucene.util.packed.PackedInts; import org.apache.lucene.util.packed.PackedLongValues; -import org.elasticsearch.index.fielddata.AbstractRandomAccessOrds; +import org.elasticsearch.index.fielddata.AbstractSortedDocValues; +import org.elasticsearch.index.fielddata.AbstractSortedSetDocValues; +import java.io.IOException; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -100,21 +101,24 @@ public Collection getChildResources() { } @Override - public RandomAccessOrds ordinals(ValuesHolder values) { + public SortedSetDocValues ordinals(ValuesHolder values) { if (multiValued) { return new MultiDocs(this, values); } else { - return (RandomAccessOrds) DocValues.singleton(new SingleDocs(this, values)); + return (SortedSetDocValues) DocValues.singleton(new SingleDocs(this, values)); } } - private static class SingleDocs extends SortedDocValues { + private static class SingleDocs extends AbstractSortedDocValues { private final int valueCount; private final PackedLongValues endOffsets; private final PackedLongValues ords; private final ValuesHolder values; + private long currentStartOffset; + private long currentEndOffset; + SingleDocs(MultiOrdinals ordinals, ValuesHolder values) { this.valueCount = (int) ordinals.valueCount; this.endOffsets = ordinals.endOffsets; @@ -123,10 +127,15 @@ private static class SingleDocs extends SortedDocValues { } @Override - public int getOrd(int docId) { - final long startOffset = docId != 0 ? endOffsets.get(docId - 1) : 0; - final long endOffset = endOffsets.get(docId); - return startOffset == endOffset ? -1 : (int) ords.get(startOffset); + public int ordValue() { + return (int) ords.get(currentStartOffset); + } + + @Override + public boolean advanceExact(int docId) throws IOException { + currentStartOffset = docId != 0 ? endOffsets.get(docId - 1) : 0; + currentEndOffset = endOffsets.get(docId); + return currentStartOffset != currentEndOffset; } @Override @@ -141,15 +150,16 @@ public int getValueCount() { } - private static class MultiDocs extends AbstractRandomAccessOrds { + private static class MultiDocs extends AbstractSortedSetDocValues { private final long valueCount; private final PackedLongValues endOffsets; private final PackedLongValues ords; - private long offset; - private int cardinality; private final ValuesHolder values; + private long currentOffset; + private long currentEndOffset; + MultiDocs(MultiOrdinals ordinals, ValuesHolder values) { this.valueCount = ordinals.valueCount; this.endOffsets = ordinals.endOffsets; @@ -163,21 +173,19 @@ public long getValueCount() { } @Override - public void doSetDocument(int docId) { - final long startOffset = docId != 0 ? endOffsets.get(docId - 1) : 0; - final long endOffset = endOffsets.get(docId); - offset = startOffset; - cardinality = (int) (endOffset - startOffset); + public boolean advanceExact(int docId) throws IOException { + currentOffset = docId != 0 ? endOffsets.get(docId - 1) : 0; + currentEndOffset = endOffsets.get(docId); + return currentOffset != currentEndOffset; } @Override - public int cardinality() { - return cardinality; - } - - @Override - public long ordAt(int index) { - return ords.get(offset + index); + public long nextOrd() throws IOException { + if (currentOffset == currentEndOffset) { + return SortedSetDocValues.NO_MORE_ORDS; + } else { + return ords.get(currentOffset++); + } } @Override diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/Ordinals.java b/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/Ordinals.java index 77c0fd6bf5ba0..7e59b708236af 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/Ordinals.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/Ordinals.java @@ -19,7 +19,7 @@ package org.elasticsearch.index.fielddata.ordinals; -import org.apache.lucene.index.RandomAccessOrds; +import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.util.Accountable; import org.apache.lucene.util.BytesRef; @@ -41,9 +41,9 @@ public BytesRef lookupOrd(long ord) { @Override public abstract long ramBytesUsed(); - public abstract RandomAccessOrds ordinals(ValuesHolder values); + public abstract SortedSetDocValues ordinals(ValuesHolder values); - public final RandomAccessOrds ordinals() { + public final SortedSetDocValues ordinals() { return ordinals(NO_VALUES); } diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/OrdinalsBuilder.java b/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/OrdinalsBuilder.java index 967d07174b9a9..516530ae73e10 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/OrdinalsBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/OrdinalsBuilder.java @@ -19,17 +19,14 @@ package org.elasticsearch.index.fielddata.ordinals; -import org.apache.lucene.index.FilteredTermsEnum; import org.apache.lucene.index.PostingsEnum; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.search.DocIdSetIterator; -import org.apache.lucene.spatial.geopoint.document.GeoPointField; import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.BitSet; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRefIterator; import org.apache.lucene.util.FixedBitSet; -import org.apache.lucene.util.LegacyNumericUtils; import org.apache.lucene.util.LongsRef; import org.apache.lucene.util.packed.GrowableWriter; import org.apache.lucene.util.packed.PackedInts; @@ -414,23 +411,6 @@ public Ordinals build() { } } - /** - * A {@link TermsEnum} that iterates only highest resolution geo prefix coded terms. - * - * @see #buildFromTerms(TermsEnum) - */ - public static TermsEnum wrapGeoPointTerms(TermsEnum termsEnum) { - return new FilteredTermsEnum(termsEnum, false) { - @Override - protected AcceptStatus accept(BytesRef term) throws IOException { - // accept only the max resolution terms - // todo is this necessary? - return GeoPointField.getPrefixCodedShift(term) == GeoPointField.PRECISION_STEP * 4 ? - AcceptStatus.YES : AcceptStatus.END; - } - }; - } - /** * Returns the maximum document ID this builder can associate with an ordinal @@ -439,37 +419,6 @@ public int maxDoc() { return maxDoc; } - /** - * A {@link TermsEnum} that iterates only full precision prefix coded 64 bit values. - * - * @see #buildFromTerms(TermsEnum) - */ - public static TermsEnum wrapNumeric64Bit(TermsEnum termsEnum) { - return new FilteredTermsEnum(termsEnum, false) { - @Override - protected AcceptStatus accept(BytesRef term) throws IOException { - // we stop accepting terms once we moved across the prefix codec terms - redundant values! - return LegacyNumericUtils.getPrefixCodedLongShift(term) == 0 ? AcceptStatus.YES : AcceptStatus.END; - } - }; - } - - /** - * A {@link TermsEnum} that iterates only full precision prefix coded 32 bit values. - * - * @see #buildFromTerms(TermsEnum) - */ - public static TermsEnum wrapNumeric32Bit(TermsEnum termsEnum) { - return new FilteredTermsEnum(termsEnum, false) { - - @Override - protected AcceptStatus accept(BytesRef term) throws IOException { - // we stop accepting terms once we moved across the prefix codec terms - redundant values! - return LegacyNumericUtils.getPrefixCodedIntShift(term) == 0 ? AcceptStatus.YES : AcceptStatus.END; - } - }; - } - /** * This method iterates all terms in the given {@link TermsEnum} and * associates each terms ordinal with the terms documents. The caller must diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/SinglePackedOrdinals.java b/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/SinglePackedOrdinals.java index f7db724c356a4..492bff0ea9ca4 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/SinglePackedOrdinals.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/SinglePackedOrdinals.java @@ -20,14 +20,15 @@ package org.elasticsearch.index.fielddata.ordinals; import org.apache.lucene.index.DocValues; -import org.apache.lucene.index.RandomAccessOrds; -import org.apache.lucene.index.SortedDocValues; +import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.util.Accountable; import org.apache.lucene.util.Accountables; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.RamUsageEstimator; import org.apache.lucene.util.packed.PackedInts; +import org.elasticsearch.index.fielddata.AbstractSortedDocValues; +import java.io.IOException; import java.util.Collection; import java.util.Collections; @@ -57,15 +58,17 @@ public Collection getChildResources() { } @Override - public RandomAccessOrds ordinals(ValuesHolder values) { - return (RandomAccessOrds) DocValues.singleton(new Docs(this, values)); + public SortedSetDocValues ordinals(ValuesHolder values) { + return (SortedSetDocValues) DocValues.singleton(new Docs(this, values)); } - private static class Docs extends SortedDocValues { + private static class Docs extends AbstractSortedDocValues { private final int maxOrd; private final PackedInts.Reader reader; private final ValuesHolder values; + + private int currentOrd; Docs(SinglePackedOrdinals parent, ValuesHolder values) { this.maxOrd = parent.valueCount; @@ -84,8 +87,14 @@ public BytesRef lookupOrd(int ord) { } @Override - public int getOrd(int docID) { - return (int) (reader.get(docID) - 1); + public int ordValue() { + return currentOrd; + } + + @Override + public boolean advanceExact(int docID) throws IOException { + currentOrd = (int) (reader.get(docID) - 1); + return currentOrd != -1; } } } diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractAtomicGeoPointFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractAtomicGeoPointFieldData.java index 9660d9f8684b8..5d6575e43783c 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractAtomicGeoPointFieldData.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractAtomicGeoPointFieldData.java @@ -59,7 +59,7 @@ public void close() { @Override public MultiGeoPointValues getGeoPointValues() { - return FieldData.emptyMultiGeoPoints(maxDoc); + return FieldData.emptyMultiGeoPoints(); } }; } diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractAtomicParentChildFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractAtomicParentChildFieldData.java index 7c03e1a794211..f82d91d37b6ce 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractAtomicParentChildFieldData.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractAtomicParentChildFieldData.java @@ -28,6 +28,7 @@ import org.elasticsearch.index.fielddata.ScriptDocValues; import org.elasticsearch.index.fielddata.SortedBinaryDocValues; +import java.io.IOException; import java.util.Collection; import java.util.Collections; import java.util.Set; @@ -48,14 +49,16 @@ public final SortedBinaryDocValues getBytesValues() { private final BytesRef[] terms = new BytesRef[2]; private int count; + private int termsCursor; @Override - public void setDocument(int docId) { + public boolean advanceExact(int docId) throws IOException { count = 0; + termsCursor = 0; for (String type : types()) { final SortedDocValues values = getOrdinalsValues(type); - final int ord = values.getOrd(docId); - if (ord >= 0) { + if (values.advanceExact(docId)) { + final int ord = values.ordValue(); terms[count++] = values.lookupOrd(ord); } } @@ -69,16 +72,17 @@ public void setDocument(int docId) { count = 1; } } + return count != 0; } @Override - public int count() { + public int docValueCount() { return count; } @Override - public BytesRef valueAt(int index) { - return terms[index]; + public BytesRef nextValue() throws IOException { + return terms[termsCursor++]; } }; } diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractIndexGeoPointFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractIndexGeoPointFieldData.java deleted file mode 100644 index bdf1bbac33235..0000000000000 --- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractIndexGeoPointFieldData.java +++ /dev/null @@ -1,117 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.fielddata.plain; - -import org.apache.lucene.search.SortField; -import org.apache.lucene.spatial.geopoint.document.GeoPointField; -import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.BytesRefIterator; -import org.apache.lucene.util.CharsRefBuilder; -import org.apache.lucene.util.LegacyNumericUtils; -import org.elasticsearch.common.Nullable; -import org.elasticsearch.common.geo.GeoPoint; -import org.elasticsearch.index.IndexSettings; -import org.elasticsearch.index.fielddata.AtomicGeoPointFieldData; -import org.elasticsearch.index.fielddata.IndexFieldData; -import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested; -import org.elasticsearch.index.fielddata.IndexFieldDataCache; -import org.elasticsearch.index.fielddata.IndexGeoPointFieldData; -import org.elasticsearch.search.MultiValueMode; - -import java.io.IOException; - -abstract class AbstractIndexGeoPointFieldData extends AbstractIndexFieldData implements IndexGeoPointFieldData { - protected abstract static class BaseGeoPointTermsEnum { - protected final BytesRefIterator termsEnum; - - protected BaseGeoPointTermsEnum(BytesRefIterator termsEnum) { - this.termsEnum = termsEnum; - } - } - - protected static class GeoPointTermsEnum extends BaseGeoPointTermsEnum { - private final GeoPointField.TermEncoding termEncoding; - protected GeoPointTermsEnum(BytesRefIterator termsEnum, GeoPointField.TermEncoding termEncoding) { - super(termsEnum); - this.termEncoding = termEncoding; - } - - public Long next() throws IOException { - final BytesRef term = termsEnum.next(); - if (term == null) { - return null; - } - if (termEncoding == GeoPointField.TermEncoding.PREFIX) { - return GeoPointField.prefixCodedToGeoCoded(term); - } else if (termEncoding == GeoPointField.TermEncoding.NUMERIC) { - return LegacyNumericUtils.prefixCodedToLong(term); - } - throw new IllegalArgumentException("GeoPoint.TermEncoding should be one of: " + GeoPointField.TermEncoding.PREFIX - + " or " + GeoPointField.TermEncoding.NUMERIC + " found: " + termEncoding); - } - } - - protected static class GeoPointTermsEnumLegacy extends BaseGeoPointTermsEnum { - private final GeoPoint next; - private final CharsRefBuilder spare; - - protected GeoPointTermsEnumLegacy(BytesRefIterator termsEnum) { - super(termsEnum); - next = new GeoPoint(); - spare = new CharsRefBuilder(); - } - - public GeoPoint next() throws IOException { - final BytesRef term = termsEnum.next(); - if (term == null) { - return null; - } - spare.copyUTF8Bytes(term); - int commaIndex = -1; - for (int i = 0; i < spare.length(); i++) { - if (spare.charAt(i) == ',') { // saves a string creation - commaIndex = i; - break; - } - } - if (commaIndex == -1) { - assert false; - return next.reset(0, 0); - } - final double lat = Double.parseDouble(new String(spare.chars(), 0, commaIndex)); - final double lon = Double.parseDouble(new String(spare.chars(), commaIndex + 1, spare.length() - (commaIndex + 1))); - return next.reset(lat, lon); - } - } - - AbstractIndexGeoPointFieldData(IndexSettings indexSettings, String fieldName, IndexFieldDataCache cache) { - super(indexSettings, fieldName, cache); - } - - @Override - public SortField sortField(@Nullable Object missingValue, MultiValueMode sortMode, Nested nested, boolean reverse) { - throw new IllegalArgumentException("can't sort on geo_point field without using specific sorting feature, like geo_distance"); - } - - @Override - protected AtomicGeoPointFieldData empty(int maxDoc) { - return AbstractAtomicGeoPointFieldData.empty(maxDoc); - } -} diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/AtomicDoubleFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/AtomicDoubleFieldData.java index 821fcc1bf291e..82ae0bb5bf1ea 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/AtomicDoubleFieldData.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/plain/AtomicDoubleFieldData.java @@ -67,7 +67,7 @@ public static AtomicNumericFieldData empty(final int maxDoc) { @Override public SortedNumericDoubleValues getDoubleValues() { - return FieldData.emptySortedNumericDoubles(maxDoc); + return FieldData.emptySortedNumericDoubles(); } @Override diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/BinaryDVAtomicFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/BinaryDVAtomicFieldData.java index c82ac69f28250..014220304d603 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/BinaryDVAtomicFieldData.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/plain/BinaryDVAtomicFieldData.java @@ -23,7 +23,6 @@ import org.apache.lucene.index.DocValues; import org.apache.lucene.index.LeafReader; import org.apache.lucene.util.Accountable; -import org.apache.lucene.util.Bits; import org.elasticsearch.index.fielddata.AtomicFieldData; import org.elasticsearch.index.fielddata.FieldData; import org.elasticsearch.index.fielddata.ScriptDocValues; @@ -49,8 +48,7 @@ public BinaryDVAtomicFieldData(LeafReader reader, String field) { public SortedBinaryDocValues getBytesValues() { try { final BinaryDocValues values = DocValues.getBinary(reader, field); - final Bits docsWithField = DocValues.getDocsWithField(reader, field); - return FieldData.singleton(values, docsWithField); + return FieldData.singleton(values); } catch (IOException e) { throw new IllegalStateException("Cannot load doc values", e); } diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/BytesBinaryDVAtomicFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/BytesBinaryDVAtomicFieldData.java index 8d43241ba75e0..b42cfefc46ab0 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/BytesBinaryDVAtomicFieldData.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/plain/BytesBinaryDVAtomicFieldData.java @@ -30,6 +30,7 @@ import org.elasticsearch.index.fielddata.ScriptDocValues; import org.elasticsearch.index.fielddata.SortedBinaryDocValues; +import java.io.IOException; import java.util.Arrays; import java.util.Collection; import java.util.Collections; @@ -59,41 +60,49 @@ public SortedBinaryDocValues getBytesValues() { int count; BytesRefBuilder[] refs = new BytesRefBuilder[0]; + int refsCursor; final ByteArrayDataInput in = new ByteArrayDataInput(); @Override - public void setDocument(int docId) { - final BytesRef bytes = values.get(docId); - in.reset(bytes.bytes, bytes.offset, bytes.length); - if (bytes.length == 0) { - count = 0; - } else { - count = in.readVInt(); - if (count > refs.length) { - final int previousLength = refs.length; - refs = Arrays.copyOf(refs, ArrayUtil.oversize(count, RamUsageEstimator.NUM_BYTES_OBJECT_REF)); - for (int i = previousLength; i < refs.length; ++i) { - refs[i] = new BytesRefBuilder(); + public boolean advanceExact(int doc) throws IOException { + refsCursor = 0; + if (values.advanceExact(doc)) { + final BytesRef bytes = values.binaryValue(); + in.reset(bytes.bytes, bytes.offset, bytes.length); + if (bytes.length == 0) { + count = 0; + } else { + count = in.readVInt(); + if (count > refs.length) { + final int previousLength = refs.length; + refs = Arrays.copyOf(refs, ArrayUtil.oversize(count, + RamUsageEstimator.NUM_BYTES_OBJECT_REF)); + for (int i = previousLength; i < refs.length; ++i) { + refs[i] = new BytesRefBuilder(); + } + } + for (int i = 0; i < count; ++i) { + final int length = in.readVInt(); + final BytesRefBuilder scratch = refs[i]; + scratch.grow(length); + in.readBytes(scratch.bytes(), 0, length); + scratch.setLength(length); } } - for (int i = 0; i < count; ++i) { - final int length = in.readVInt(); - final BytesRefBuilder scratch = refs[i]; - scratch.grow(length); - in.readBytes(scratch.bytes(), 0, length); - scratch.setLength(length); - } + return true; + } else { + return false; } } @Override - public int count() { + public int docValueCount() { return count; } @Override - public BytesRef valueAt(int index) { - return refs[index].get(); + public BytesRef nextValue() throws IOException { + return refs[refsCursor++].get(); } }; diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/DocValuesIndexFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/DocValuesIndexFieldData.java index 5378a201d44ef..4a066328bc2a9 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/DocValuesIndexFieldData.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/plain/DocValuesIndexFieldData.java @@ -21,7 +21,6 @@ import org.apache.logging.log4j.Logger; import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.RandomAccessOrds; import org.apache.lucene.index.SortedSetDocValues; import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.index.Index; diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/GeoPointDVAtomicFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/GeoPointDVAtomicFieldData.java index a71ea3a552e1c..11cb1c7b8af71 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/GeoPointDVAtomicFieldData.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/plain/GeoPointDVAtomicFieldData.java @@ -26,6 +26,7 @@ import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.index.fielddata.MultiGeoPointValues; +import java.io.IOException; import java.util.Arrays; import java.util.Collection; import java.util.Collections; @@ -58,32 +59,39 @@ public void close() { public MultiGeoPointValues getGeoPointValues() { return new MultiGeoPointValues() { GeoPoint[] points = new GeoPoint[0]; + private int pointsCursor = 0; private int count = 0; @Override - public void setDocument(int docId) { - values.setDocument(docId); - count = values.count(); - if (count > points.length) { - final int previousLength = points.length; - points = Arrays.copyOf(points, ArrayUtil.oversize(count, RamUsageEstimator.NUM_BYTES_OBJECT_REF)); - for (int i = previousLength; i < points.length; ++i) { - points[i] = new GeoPoint(Double.NaN, Double.NaN); + public boolean advanceExact(int doc) throws IOException { + if (values.advanceExact(doc)) { + count = values.docValueCount(); + pointsCursor = 0; + if (count > points.length) { + final int previousLength = points.length; + points = Arrays.copyOf(points, + ArrayUtil.oversize(count, RamUsageEstimator.NUM_BYTES_OBJECT_REF)); + for (int i = previousLength; i < points.length; ++i) { + points[i] = new GeoPoint(Double.NaN, Double.NaN); + } } - } - for (int i=0; i getChildResources() { } @Override - public RandomAccessOrds getOrdinalsValues() { + public SortedSetDocValues getOrdinalsValues() { final BytesRef term = new BytesRef(index); - final SortedDocValues sortedValues = new SortedDocValues() { + final SortedDocValues sortedValues = new AbstractSortedDocValues() { @Override public BytesRef lookupOrd(int ord) { @@ -90,11 +92,16 @@ public int getValueCount() { } @Override - public int getOrd(int docID) { + public int ordValue() { return 0; } + + @Override + public boolean advanceExact(int target) throws IOException { + return true; + } }; - return (RandomAccessOrds) DocValues.singleton(sortedValues); + return (SortedSetDocValues) DocValues.singleton(sortedValues); } @Override diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/LatLonPointDVAtomicFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/LatLonPointDVAtomicFieldData.java index d11a79c255654..99cdbc451ae2b 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/LatLonPointDVAtomicFieldData.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/plain/LatLonPointDVAtomicFieldData.java @@ -26,6 +26,7 @@ import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.index.fielddata.MultiGeoPointValues; +import java.io.IOException; import java.util.Arrays; import java.util.Collection; import java.util.Collections; @@ -57,34 +58,41 @@ public void close() { public MultiGeoPointValues getGeoPointValues() { return new MultiGeoPointValues() { GeoPoint[] points = new GeoPoint[0]; + private int pointsCursor; private int count = 0; @Override - public void setDocument(int docId) { - values.setDocument(docId); - count = values.count(); - if (count > points.length) { - final int previousLength = points.length; - points = Arrays.copyOf(points, ArrayUtil.oversize(count, RamUsageEstimator.NUM_BYTES_OBJECT_REF)); - for (int i = previousLength; i < points.length; ++i) { - points[i] = new GeoPoint(Double.NaN, Double.NaN); + public boolean advanceExact(int doc) throws IOException { + if (values.advanceExact(doc)) { + count = values.docValueCount(); + if (count > points.length) { + final int previousLength = points.length; + points = Arrays.copyOf(points, + ArrayUtil.oversize(count, RamUsageEstimator.NUM_BYTES_OBJECT_REF)); + for (int i = previousLength; i < points.length; ++i) { + points[i] = new GeoPoint(Double.NaN, Double.NaN); + } } - } - long encoded; - for (int i=0; i>> 32)), GeoEncodingUtils.decodeLongitude((int)encoded)); + long encoded; + for (int i = 0; i < count; ++i) { + encoded = values.nextValue(); + points[i].reset(GeoEncodingUtils.decodeLatitude((int) (encoded >>> 32)), + GeoEncodingUtils.decodeLongitude((int) encoded)); + } + return true; + } else { + return false; } } @Override - public int count() { + public int docValueCount() { return count; } @Override - public GeoPoint valueAt(int index) { - return points[index]; + public GeoPoint nextValue() { + return points[pointsCursor++]; } }; } diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/PagedBytesAtomicFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/PagedBytesAtomicFieldData.java index 02e7da8886911..c9e81ca4f2dc1 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/PagedBytesAtomicFieldData.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/plain/PagedBytesAtomicFieldData.java @@ -18,7 +18,7 @@ */ package org.elasticsearch.index.fielddata.plain; -import org.apache.lucene.index.RandomAccessOrds; +import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.util.Accountable; import org.apache.lucene.util.Accountables; import org.apache.lucene.util.BytesRef; @@ -68,7 +68,7 @@ public Collection getChildResources() { } @Override - public RandomAccessOrds getOrdinalsValues() { + public SortedSetDocValues getOrdinalsValues() { return ordinals.ordinals(new ValuesHolder(bytes, termOrdToBytesOffset)); } diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/ParentChildIndexFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/ParentChildIndexFieldData.java index ee451ff023375..647e275664c53 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/ParentChildIndexFieldData.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/plain/ParentChildIndexFieldData.java @@ -40,6 +40,7 @@ import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.fielddata.AbstractSortedDocValues; import org.elasticsearch.index.fielddata.AtomicParentChildFieldData; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested; @@ -254,10 +255,10 @@ public SortedDocValues getOrdinalsValues(String type) { return segmentValues; } final LongValues globalOrds = ordMap.getGlobalOrds(segmentIndex); - return new SortedDocValues() { + return new AbstractSortedDocValues() { @Override - public BytesRef lookupOrd(int ord) { + public BytesRef lookupOrd(int ord) throws IOException { final int segmentIndex = ordMap.getFirstSegmentNumber(ord); final int segmentOrd = (int) ordMap.getFirstSegmentOrd(ord); return allSegmentValues[segmentIndex].lookupOrd(segmentOrd); @@ -269,14 +270,13 @@ public int getValueCount() { } @Override - public int getOrd(int docID) { - final int segmentOrd = segmentValues.getOrd(docID); - // TODO: is there a way we can get rid of this branch? - if (segmentOrd >= 0) { - return (int) globalOrds.get(segmentOrd); - } else { - return segmentOrd; - } + public int ordValue() { + return (int) globalOrds.get(segmentValues.ordValue()); + } + + @Override + public boolean advanceExact(int target) throws IOException { + return segmentValues.advanceExact(target); } }; } @@ -313,7 +313,7 @@ public class GlobalFieldData implements IndexParentChildFieldData, Accountable { private final Map ordinalMapPerType; GlobalFieldData(IndexReader reader, AtomicParentChildFieldData[] fielddata, long ramBytesUsed, Map ordinalMapPerType) { - this.coreCacheKey = reader.getCoreCacheKey(); + this.coreCacheKey = reader.getReaderCacheHelper().getKey(); this.leaves = reader.leaves(); this.ramBytesUsed = ramBytesUsed; this.fielddata = fielddata; @@ -327,7 +327,8 @@ public String getFieldName() { @Override public AtomicParentChildFieldData load(LeafReaderContext context) { - assert context.reader().getCoreCacheKey() == leaves.get(context.ord).reader().getCoreCacheKey(); + assert context.reader().getCoreCacheHelper().getKey() == leaves.get(context.ord) + .reader().getCoreCacheHelper().getKey(); return fielddata[context.ord]; } @@ -363,7 +364,7 @@ public Collection getChildResources() { @Override public IndexParentChildFieldData loadGlobal(DirectoryReader indexReader) { - if (indexReader.getCoreCacheKey() == coreCacheKey) { + if (indexReader.getReaderCacheHelper().getKey() == coreCacheKey) { return this; } throw new IllegalStateException(); diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/SortedNumericDVIndexFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/SortedNumericDVIndexFieldData.java index 82f3de0f72738..9be850651fe3c 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/SortedNumericDVIndexFieldData.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/plain/SortedNumericDVIndexFieldData.java @@ -209,7 +209,7 @@ public SortedNumericDoubleValues getDoubleValues() { NumericDocValues single = DocValues.unwrapSingleton(raw); if (single != null) { - return FieldData.singleton(new SingleHalfFloatValues(single), DocValues.unwrapSingletonBits(raw)); + return FieldData.singleton(new SingleHalfFloatValues(single)); } else { return new MultiHalfFloatValues(raw); } @@ -235,8 +235,13 @@ static final class SingleHalfFloatValues extends NumericDoubleValues { } @Override - public double get(int docID) { - return HalfFloatPoint.sortableShortToHalfFloat((short) in.get(docID)); + public double doubleValue() throws IOException { + return HalfFloatPoint.sortableShortToHalfFloat((short) in.longValue()); + } + + @Override + public boolean advanceExact(int doc) throws IOException { + return in.advanceExact(doc); } } @@ -251,18 +256,18 @@ static final class MultiHalfFloatValues extends SortedNumericDoubleValues { } @Override - public void setDocument(int doc) { - in.setDocument(doc); + public boolean advanceExact(int target) throws IOException { + return in.advanceExact(target); } @Override - public double valueAt(int index) { - return HalfFloatPoint.sortableShortToHalfFloat((short) in.valueAt(index)); + public double nextValue() throws IOException { + return HalfFloatPoint.sortableShortToHalfFloat((short) in.nextValue()); } @Override - public int count() { - return in.count(); + public int docValueCount() { + return in.docValueCount(); } } @@ -298,7 +303,7 @@ public SortedNumericDoubleValues getDoubleValues() { NumericDocValues single = DocValues.unwrapSingleton(raw); if (single != null) { - return FieldData.singleton(new SingleFloatValues(single), DocValues.unwrapSingletonBits(raw)); + return FieldData.singleton(new SingleFloatValues(single)); } else { return new MultiFloatValues(raw); } @@ -324,8 +329,13 @@ static final class SingleFloatValues extends NumericDoubleValues { } @Override - public double get(int docID) { - return NumericUtils.sortableIntToFloat((int) in.get(docID)); + public double doubleValue() throws IOException { + return NumericUtils.sortableIntToFloat((int) in.longValue()); + } + + @Override + public boolean advanceExact(int doc) throws IOException { + return in.advanceExact(doc); } } @@ -340,18 +350,18 @@ static final class MultiFloatValues extends SortedNumericDoubleValues { } @Override - public void setDocument(int doc) { - in.setDocument(doc); + public boolean advanceExact(int target) throws IOException { + return in.advanceExact(target); } @Override - public double valueAt(int index) { - return NumericUtils.sortableIntToFloat((int) in.valueAt(index)); + public double nextValue() throws IOException { + return NumericUtils.sortableIntToFloat((int) in.nextValue()); } @Override - public int count() { - return in.count(); + public int docValueCount() { + return in.docValueCount(); } } From 0a85324f06fcb47c91ca70114f255c2856e11a82 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Thu, 6 Apr 2017 17:02:50 +0200 Subject: [PATCH 31/87] Fix compile errors in the fielddata package under core/tests. --- .../AbstractFieldDataImplTestCase.java | 47 ++++++++------- .../AbstractGeoFieldDataTestCase.java | 21 +++---- .../AbstractStringFieldDataTestCase.java | 60 +++++++------------ .../fielddata/BinaryDVFieldDataTests.java | 26 ++++---- .../index/fielddata/FieldDataTests.java | 52 ++++++++++------ .../index/fielddata/FilterFieldDataTests.java | 12 ++-- .../fielddata/ParentChildFieldDataTests.java | 57 +++++++++--------- .../fielddata/ScriptDocValuesDatesTests.java | 14 +++-- .../ScriptDocValuesGeoPointsTests.java | 11 ++-- .../fielddata/ScriptDocValuesLongsTests.java | 13 ++-- .../ordinals/MultiOrdinalsTests.java | 48 ++++++--------- .../ordinals/SingleOrdinalsTests.java | 7 ++- .../plain/HalfFloatFielddataTests.java | 14 ++--- 13 files changed, 188 insertions(+), 194 deletions(-) diff --git a/core/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataImplTestCase.java b/core/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataImplTestCase.java index c96292d90e5fb..6973d3b26703e 100644 --- a/core/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataImplTestCase.java +++ b/core/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataImplTestCase.java @@ -31,6 +31,7 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.search.MultiValueMode; +import java.io.IOException; import java.util.List; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThanOrEqualTo; @@ -77,8 +78,8 @@ public void testDeletedDocs() throws Exception { AtomicFieldData fieldData = indexFieldData.load(readerContext); SortedBinaryDocValues values = fieldData.getBytesValues(); for (int i = 0; i < readerContext.reader().maxDoc(); ++i) { - values.setDocument(i); - assertThat(values.count(), greaterThanOrEqualTo(1)); + assertTrue(values.advanceExact(i)); + assertThat(values.docValueCount(), greaterThanOrEqualTo(1)); } } } @@ -93,15 +94,15 @@ public void testSingleValueAllSet() throws Exception { SortedBinaryDocValues bytesValues = fieldData.getBytesValues(); - bytesValues.setDocument(0); - assertThat(bytesValues.count(), equalTo(1)); - assertThat(bytesValues.valueAt(0), equalTo(new BytesRef(two()))); - bytesValues.setDocument(1); - assertThat(bytesValues.count(), equalTo(1)); - assertThat(bytesValues.valueAt(0), equalTo(new BytesRef(one()))); - bytesValues.setDocument(2); - assertThat(bytesValues.count(), equalTo(1)); - assertThat(bytesValues.valueAt(0), equalTo(new BytesRef(three()))); + assertTrue(bytesValues.advanceExact(0)); + assertThat(bytesValues.docValueCount(), equalTo(1)); + assertThat(bytesValues.nextValue(), equalTo(new BytesRef(two()))); + assertTrue(bytesValues.advanceExact(1)); + assertThat(bytesValues.docValueCount(), equalTo(1)); + assertThat(bytesValues.nextValue(), equalTo(new BytesRef(one()))); + assertTrue(bytesValues.advanceExact(2)); + assertThat(bytesValues.docValueCount(), equalTo(1)); + assertThat(bytesValues.nextValue(), equalTo(new BytesRef(three()))); assertValues(bytesValues, 0, two()); assertValues(bytesValues, 1, one()); @@ -132,19 +133,23 @@ public void testSingleValueAllSet() throws Exception { protected abstract void fillSingleValueWithMissing() throws Exception; - public void assertValues(SortedBinaryDocValues values, int docId, BytesRef... actualValues) { - values.setDocument(docId); - assertThat(values.count(), equalTo(actualValues.length)); - for (int i = 0; i < actualValues.length; i++) { - assertThat(values.valueAt(i), equalTo(actualValues[i])); + public void assertValues(SortedBinaryDocValues values, int docId, BytesRef... actualValues) throws IOException { + assertEquals(actualValues.length > 0, values.advanceExact(docId)); + if (actualValues.length > 0) { + assertThat(values.docValueCount(), equalTo(actualValues.length)); + for (int i = 0; i < actualValues.length; i++) { + assertThat(values.nextValue(), equalTo(actualValues[i])); + } } } - public void assertValues(SortedBinaryDocValues values, int docId, String... actualValues) { - values.setDocument(docId); - assertThat(values.count(), equalTo(actualValues.length)); - for (int i = 0; i < actualValues.length; i++) { - assertThat(values.valueAt(i), equalTo(new BytesRef(actualValues[i]))); + public void assertValues(SortedBinaryDocValues values, int docId, String... actualValues) throws IOException { + assertEquals(actualValues.length > 0, values.advanceExact(docId)); + if (actualValues.length > 0) { + assertThat(values.docValueCount(), equalTo(actualValues.length)); + for (int i = 0; i < actualValues.length; i++) { + assertThat(values.nextValue(), equalTo(new BytesRef(actualValues[i]))); + } } } diff --git a/core/src/test/java/org/elasticsearch/index/fielddata/AbstractGeoFieldDataTestCase.java b/core/src/test/java/org/elasticsearch/index/fielddata/AbstractGeoFieldDataTestCase.java index 21d816e83ea7a..a46fd68a291ab 100644 --- a/core/src/test/java/org/elasticsearch/index/fielddata/AbstractGeoFieldDataTestCase.java +++ b/core/src/test/java/org/elasticsearch/index/fielddata/AbstractGeoFieldDataTestCase.java @@ -25,10 +25,10 @@ import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.common.geo.GeoUtils; +import java.io.IOException; + import static org.elasticsearch.test.geo.RandomShapeGenerator.randomPoint; import static org.hamcrest.Matchers.allOf; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.lessThanOrEqualTo; @@ -71,23 +71,20 @@ public void testSortMultiValuesFields() { assumeFalse("Only test on non geo_point fields", getFieldDataType().equals("geo_point")); } - protected void assertValues(MultiGeoPointValues values, int docId) { + protected void assertValues(MultiGeoPointValues values, int docId) throws IOException { assertValues(values, docId, false); } - protected void assertMissing(MultiGeoPointValues values, int docId) { + protected void assertMissing(MultiGeoPointValues values, int docId) throws IOException { assertValues(values, docId, true); } - private void assertValues(MultiGeoPointValues values, int docId, boolean missing) { - values.setDocument(docId); - int docCount = values.count(); - if (missing) { - assertThat(docCount, equalTo(0)); - } else { - assertThat(docCount, greaterThan(0)); + private void assertValues(MultiGeoPointValues values, int docId, boolean missing) throws IOException { + assertEquals(missing == false, values.advanceExact(docId)); + if (missing == false) { + final int docCount = values.docValueCount(); for (int i = 0; i < docCount; ++i) { - final GeoPoint point = values.valueAt(i); + final GeoPoint point = values.nextValue(); assertThat(point.lat(), allOf(greaterThanOrEqualTo(GeoUtils.MIN_LAT), lessThanOrEqualTo(GeoUtils.MAX_LAT))); assertThat(point.lon(), allOf(greaterThanOrEqualTo(GeoUtils.MIN_LON), lessThanOrEqualTo(GeoUtils.MAX_LON))); } diff --git a/core/src/test/java/org/elasticsearch/index/fielddata/AbstractStringFieldDataTestCase.java b/core/src/test/java/org/elasticsearch/index/fielddata/AbstractStringFieldDataTestCase.java index c24162788789c..aebb0a802edca 100644 --- a/core/src/test/java/org/elasticsearch/index/fielddata/AbstractStringFieldDataTestCase.java +++ b/core/src/test/java/org/elasticsearch/index/fielddata/AbstractStringFieldDataTestCase.java @@ -20,6 +20,7 @@ package org.elasticsearch.index.fielddata; import com.carrotsearch.randomizedtesting.generators.RandomPicks; + import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.Field.Store; @@ -27,7 +28,7 @@ import org.apache.lucene.document.StringField; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.RandomAccessOrds; +import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.index.Term; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.search.ConstantScoreQuery; @@ -48,8 +49,6 @@ import org.apache.lucene.util.UnicodeUtil; import org.elasticsearch.common.lucene.index.ElasticsearchDirectoryReader; import org.elasticsearch.common.lucene.search.Queries; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource; import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested; import org.elasticsearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource; import org.elasticsearch.index.fielddata.ordinals.GlobalOrdinalsIndexFieldData; @@ -453,19 +452,6 @@ public void testNestedSorting(MultiValueMode sortMode) throws IOException { searcher.getIndexReader().close(); } - private void assertIteratorConsistentWithRandomAccess(RandomAccessOrds ords, int maxDoc) { - for (int doc = 0; doc < maxDoc; ++doc) { - ords.setDocument(doc); - final int cardinality = ords.cardinality(); - for (int i = 0; i < cardinality; ++i) { - assertEquals(ords.nextOrd(), ords.ordAt(i)); - } - for (int i = 0; i < 3; ++i) { - assertEquals(ords.nextOrd(), -1); - } - } - } - public void testGlobalOrdinals() throws Exception { fillExtendedMvSet(); refreshReader(); @@ -477,31 +463,29 @@ public void testGlobalOrdinals() throws Exception { assertThat(globalOrdinals, instanceOf(GlobalOrdinalsIndexFieldData.class)); LeafReaderContext leaf = topLevelReader.leaves().get(0); AtomicOrdinalsFieldData afd = globalOrdinals.load(leaf); - RandomAccessOrds values = afd.getOrdinalsValues(); - assertIteratorConsistentWithRandomAccess(values, leaf.reader().maxDoc()); - values.setDocument(0); - assertThat(values.cardinality(), equalTo(2)); + SortedSetDocValues values = afd.getOrdinalsValues(); + assertTrue(values.advanceExact(0)); long ord = values.nextOrd(); assertThat(ord, equalTo(3L)); assertThat(values.lookupOrd(ord).utf8ToString(), equalTo("02")); ord = values.nextOrd(); assertThat(ord, equalTo(5L)); assertThat(values.lookupOrd(ord).utf8ToString(), equalTo("04")); - values.setDocument(1); - assertThat(values.cardinality(), equalTo(0)); - values.setDocument(2); - assertThat(values.cardinality(), equalTo(1)); + ord = values.nextOrd(); + assertThat(ord, equalTo(SortedSetDocValues.NO_MORE_ORDS)); + assertFalse(values.advanceExact(1)); + assertTrue(values.advanceExact(2)); ord = values.nextOrd(); assertThat(ord, equalTo(4L)); assertThat(values.lookupOrd(ord).utf8ToString(), equalTo("03")); + ord = values.nextOrd(); + assertThat(ord, equalTo(SortedSetDocValues.NO_MORE_ORDS)); // Second segment leaf = topLevelReader.leaves().get(1); afd = globalOrdinals.load(leaf); values = afd.getOrdinalsValues(); - assertIteratorConsistentWithRandomAccess(values, leaf.reader().maxDoc()); - values.setDocument(0); - assertThat(values.cardinality(), equalTo(3)); + assertTrue(values.advanceExact(0)); ord = values.nextOrd(); assertThat(ord, equalTo(5L)); assertThat(values.lookupOrd(ord).utf8ToString(), equalTo("04")); @@ -511,8 +495,9 @@ public void testGlobalOrdinals() throws Exception { ord = values.nextOrd(); assertThat(ord, equalTo(7L)); assertThat(values.lookupOrd(ord).utf8ToString(), equalTo("06")); - values.setDocument(1); - assertThat(values.cardinality(), equalTo(3)); + ord = values.nextOrd(); + assertThat(ord, equalTo(SortedSetDocValues.NO_MORE_ORDS)); + assertTrue(values.advanceExact(1)); ord = values.nextOrd(); assertThat(ord, equalTo(7L)); assertThat(values.lookupOrd(ord).utf8ToString(), equalTo("06")); @@ -522,10 +507,10 @@ public void testGlobalOrdinals() throws Exception { ord = values.nextOrd(); assertThat(ord, equalTo(9L)); assertThat(values.lookupOrd(ord).utf8ToString(), equalTo("08")); - values.setDocument(2); - assertThat(values.cardinality(), equalTo(0)); - values.setDocument(3); - assertThat(values.cardinality(), equalTo(3)); + ord = values.nextOrd(); + assertThat(ord, equalTo(SortedSetDocValues.NO_MORE_ORDS)); + assertFalse(values.advanceExact(2)); + assertTrue(values.advanceExact(3)); ord = values.nextOrd(); assertThat(ord, equalTo(9L)); assertThat(values.lookupOrd(ord).utf8ToString(), equalTo("08")); @@ -535,15 +520,14 @@ public void testGlobalOrdinals() throws Exception { ord = values.nextOrd(); assertThat(ord, equalTo(11L)); assertThat(values.lookupOrd(ord).utf8ToString(), equalTo("10")); + ord = values.nextOrd(); + assertThat(ord, equalTo(SortedSetDocValues.NO_MORE_ORDS)); // Third segment leaf = topLevelReader.leaves().get(2); afd = globalOrdinals.load(leaf); values = afd.getOrdinalsValues(); - assertIteratorConsistentWithRandomAccess(values, leaf.reader().maxDoc()); - values.setDocument(0); - values.setDocument(0); - assertThat(values.cardinality(), equalTo(3)); + assertTrue(values.advanceExact(0)); ord = values.nextOrd(); assertThat(ord, equalTo(0L)); assertThat(values.lookupOrd(ord).utf8ToString(), equalTo("!08")); @@ -553,6 +537,8 @@ public void testGlobalOrdinals() throws Exception { ord = values.nextOrd(); assertThat(ord, equalTo(2L)); assertThat(values.lookupOrd(ord).utf8ToString(), equalTo("!10")); + ord = values.nextOrd(); + assertThat(ord, equalTo(SortedSetDocValues.NO_MORE_ORDS)); } public void testTermsEnum() throws Exception { diff --git a/core/src/test/java/org/elasticsearch/index/fielddata/BinaryDVFieldDataTests.java b/core/src/test/java/org/elasticsearch/index/fielddata/BinaryDVFieldDataTests.java index 70c1486fe1f03..64ab07442b4c2 100644 --- a/core/src/test/java/org/elasticsearch/index/fielddata/BinaryDVFieldDataTests.java +++ b/core/src/test/java/org/elasticsearch/index/fielddata/BinaryDVFieldDataTests.java @@ -23,7 +23,6 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.compress.CompressedXContent; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.CollectionUtils; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; @@ -87,23 +86,22 @@ public void testDocValue() throws Exception { SortedBinaryDocValues bytesValues = fieldData.getBytesValues(); CollectionUtils.sortAndDedup(bytesList1); - bytesValues.setDocument(0); - assertThat(bytesValues.count(), equalTo(2)); - assertThat(bytesValues.valueAt(0), equalTo(new BytesRef(bytesList1.get(0)))); - assertThat(bytesValues.valueAt(1), equalTo(new BytesRef(bytesList1.get(1)))); + assertTrue(bytesValues.advanceExact(0)); + assertThat(bytesValues.docValueCount(), equalTo(2)); + assertThat(bytesValues.nextValue(), equalTo(new BytesRef(bytesList1.get(0)))); + assertThat(bytesValues.nextValue(), equalTo(new BytesRef(bytesList1.get(1)))); - bytesValues.setDocument(1); - assertThat(bytesValues.count(), equalTo(1)); - assertThat(bytesValues.valueAt(0), equalTo(new BytesRef(bytes1))); + assertTrue(bytesValues.advanceExact(0)); + assertThat(bytesValues.docValueCount(), equalTo(1)); + assertThat(bytesValues.nextValue(), equalTo(new BytesRef(bytes1))); - bytesValues.setDocument(2); - assertThat(bytesValues.count(), equalTo(0)); + assertFalse(bytesValues.advanceExact(2)); CollectionUtils.sortAndDedup(bytesList2); - bytesValues.setDocument(3); - assertThat(bytesValues.count(), equalTo(2)); - assertThat(bytesValues.valueAt(0), equalTo(new BytesRef(bytesList2.get(0)))); - assertThat(bytesValues.valueAt(1), equalTo(new BytesRef(bytesList2.get(1)))); + assertTrue(bytesValues.advanceExact(3)); + assertThat(bytesValues.docValueCount(), equalTo(2)); + assertThat(bytesValues.nextValue(), equalTo(new BytesRef(bytesList2.get(0)))); + assertThat(bytesValues.nextValue(), equalTo(new BytesRef(bytesList2.get(1)))); } } diff --git a/core/src/test/java/org/elasticsearch/index/fielddata/FieldDataTests.java b/core/src/test/java/org/elasticsearch/index/fielddata/FieldDataTests.java index 33ddc80868a6d..0c01d4064d38d 100644 --- a/core/src/test/java/org/elasticsearch/index/fielddata/FieldDataTests.java +++ b/core/src/test/java/org/elasticsearch/index/fielddata/FieldDataTests.java @@ -25,83 +25,97 @@ import org.apache.lucene.util.NumericUtils; import org.elasticsearch.test.ESTestCase; +import java.io.IOException; + public class FieldDataTests extends ESTestCase { - public void testSortableLongBitsToDoubles() { + public void testSortableLongBitsToDoubles() throws IOException { final double value = randomDouble(); final long valueBits = NumericUtils.doubleToSortableLong(value); - NumericDocValues values = new NumericDocValues() { + NumericDocValues values = new AbstractNumericDocValues() { + @Override + public boolean advanceExact(int target) throws IOException { + return true; + } @Override - public long get(int docID) { + public long longValue() throws IOException { return valueBits; } }; - SortedNumericDoubleValues asMultiDoubles = FieldData.sortableLongBitsToDoubles(DocValues.singleton(values, null)); + SortedNumericDoubleValues asMultiDoubles = FieldData.sortableLongBitsToDoubles(DocValues.singleton(values)); NumericDoubleValues asDoubles = FieldData.unwrapSingleton(asMultiDoubles); assertNotNull(asDoubles); - assertEquals(value, asDoubles.get(0), 0); + assertTrue(asDoubles.advanceExact(0)); + assertEquals(value, asDoubles.doubleValue(), 0); NumericDocValues backToLongs = DocValues.unwrapSingleton(FieldData.toSortableLongBits(asMultiDoubles)); assertSame(values, backToLongs); - SortedNumericDocValues multiValues = new SortedNumericDocValues() { + SortedNumericDocValues multiValues = new AbstractSortedNumericDocValues() { @Override - public long valueAt(int index) { - return valueBits; + public boolean advanceExact(int target) throws IOException { + return true; } @Override - public void setDocument(int doc) { + public long nextValue() { + return valueBits; } @Override - public int count() { + public int docValueCount() { return 1; } }; asMultiDoubles = FieldData.sortableLongBitsToDoubles(multiValues); - assertEquals(value, asMultiDoubles.valueAt(0), 0); + assertEquals(value, asMultiDoubles.nextValue(), 0); assertSame(multiValues, FieldData.toSortableLongBits(asMultiDoubles)); } - public void testDoublesToSortableLongBits() { + public void testDoublesToSortableLongBits() throws IOException { final double value = randomDouble(); final long valueBits = NumericUtils.doubleToSortableLong(value); NumericDoubleValues values = new NumericDoubleValues() { @Override - public double get(int docID) { + public boolean advanceExact(int doc) throws IOException { + return true; + } + @Override + public double doubleValue() { return value; } }; - SortedNumericDocValues asMultiLongs = FieldData.toSortableLongBits(FieldData.singleton(values, null)); + SortedNumericDocValues asMultiLongs = FieldData.toSortableLongBits(FieldData.singleton(values)); NumericDocValues asLongs = DocValues.unwrapSingleton(asMultiLongs); assertNotNull(asLongs); - assertEquals(valueBits, asLongs.get(0)); + assertTrue(asLongs.advanceExact(0)); + assertEquals(valueBits, asLongs.longValue()); SortedNumericDoubleValues multiValues = new SortedNumericDoubleValues() { @Override - public double valueAt(int index) { + public double nextValue() { return value; } @Override - public void setDocument(int doc) { + public boolean advanceExact(int target) throws IOException { + return true; } @Override - public int count() { + public int docValueCount() { return 1; } }; asMultiLongs = FieldData.toSortableLongBits(multiValues); - assertEquals(valueBits, asMultiLongs.valueAt(0)); + assertEquals(valueBits, asMultiLongs.nextValue()); assertSame(multiValues, FieldData.sortableLongBitsToDoubles(asMultiLongs)); } } diff --git a/core/src/test/java/org/elasticsearch/index/fielddata/FilterFieldDataTests.java b/core/src/test/java/org/elasticsearch/index/fielddata/FilterFieldDataTests.java index 33ecd57b130f1..19725aca523ac 100644 --- a/core/src/test/java/org/elasticsearch/index/fielddata/FilterFieldDataTests.java +++ b/core/src/test/java/org/elasticsearch/index/fielddata/FilterFieldDataTests.java @@ -22,11 +22,11 @@ import org.apache.lucene.document.Field; import org.apache.lucene.document.StringField; import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.RandomAccessOrds; +import org.apache.lucene.index.SortedSetDocValues; import org.elasticsearch.index.mapper.ContentPath; import org.elasticsearch.index.mapper.MappedFieldType; -import org.elasticsearch.index.mapper.TextFieldMapper; import org.elasticsearch.index.mapper.Mapper.BuilderContext; +import org.elasticsearch.index.mapper.TextFieldMapper; import java.util.List; import java.util.Random; @@ -71,7 +71,7 @@ public void testFilterByFrequency() throws Exception { IndexOrdinalsFieldData fieldData = ifdService.getForField(ft); for (LeafReaderContext context : contexts) { AtomicOrdinalsFieldData loadDirect = fieldData.loadDirect(context); - RandomAccessOrds bytesValues = loadDirect.getOrdinalsValues(); + SortedSetDocValues bytesValues = loadDirect.getOrdinalsValues(); assertThat(2L, equalTo(bytesValues.getValueCount())); assertThat(bytesValues.lookupOrd(0).utf8ToString(), equalTo("10")); assertThat(bytesValues.lookupOrd(1).utf8ToString(), equalTo("100")); @@ -86,7 +86,7 @@ public void testFilterByFrequency() throws Exception { IndexOrdinalsFieldData fieldData = ifdService.getForField(ft); for (LeafReaderContext context : contexts) { AtomicOrdinalsFieldData loadDirect = fieldData.loadDirect(context); - RandomAccessOrds bytesValues = loadDirect.getOrdinalsValues(); + SortedSetDocValues bytesValues = loadDirect.getOrdinalsValues(); assertThat(1L, equalTo(bytesValues.getValueCount())); assertThat(bytesValues.lookupOrd(0).utf8ToString(), equalTo("5")); } @@ -101,7 +101,7 @@ public void testFilterByFrequency() throws Exception { IndexOrdinalsFieldData fieldData = ifdService.getForField(ft); for (LeafReaderContext context : contexts) { AtomicOrdinalsFieldData loadDirect = fieldData.loadDirect(context); - RandomAccessOrds bytesValues = loadDirect.getOrdinalsValues(); + SortedSetDocValues bytesValues = loadDirect.getOrdinalsValues(); assertThat(2L, equalTo(bytesValues.getValueCount())); assertThat(bytesValues.lookupOrd(0).utf8ToString(), equalTo("10")); assertThat(bytesValues.lookupOrd(1).utf8ToString(), equalTo("100")); @@ -117,7 +117,7 @@ public void testFilterByFrequency() throws Exception { IndexOrdinalsFieldData fieldData = ifdService.getForField(ft); for (LeafReaderContext context : contexts) { AtomicOrdinalsFieldData loadDirect = fieldData.loadDirect(context); - RandomAccessOrds bytesValues = loadDirect.getOrdinalsValues(); + SortedSetDocValues bytesValues = loadDirect.getOrdinalsValues(); assertThat(2L, equalTo(bytesValues.getValueCount())); assertThat(bytesValues.lookupOrd(0).utf8ToString(), equalTo("10")); assertThat(bytesValues.lookupOrd(1).utf8ToString(), equalTo("100")); diff --git a/core/src/test/java/org/elasticsearch/index/fielddata/ParentChildFieldDataTests.java b/core/src/test/java/org/elasticsearch/index/fielddata/ParentChildFieldDataTests.java index f426e5433c6f3..f7bdc8efed136 100644 --- a/core/src/test/java/org/elasticsearch/index/fielddata/ParentChildFieldDataTests.java +++ b/core/src/test/java/org/elasticsearch/index/fielddata/ParentChildFieldDataTests.java @@ -132,40 +132,39 @@ public void testGetBytesValues() throws Exception { AtomicFieldData fieldData = indexFieldData.load(readerContext); SortedBinaryDocValues bytesValues = fieldData.getBytesValues(); - bytesValues.setDocument(0); - assertThat(bytesValues.count(), equalTo(1)); - assertThat(bytesValues.valueAt(0).utf8ToString(), equalTo("1")); + assertTrue(bytesValues.advanceExact(0)); + assertThat(bytesValues.docValueCount(), equalTo(1)); + assertThat(bytesValues.nextValue().utf8ToString(), equalTo("1")); - bytesValues.setDocument(1); - assertThat(bytesValues.count(), equalTo(2)); - assertThat(bytesValues.valueAt(0).utf8ToString(), equalTo("1")); - assertThat(bytesValues.valueAt(1).utf8ToString(), equalTo("2")); + assertTrue(bytesValues.advanceExact(1)); + assertThat(bytesValues.docValueCount(), equalTo(2)); + assertThat(bytesValues.nextValue().utf8ToString(), equalTo("1")); + assertThat(bytesValues.nextValue().utf8ToString(), equalTo("2")); - bytesValues.setDocument(2); - assertThat(bytesValues.count(), equalTo(2)); - assertThat(bytesValues.valueAt(0).utf8ToString(), equalTo("1")); - assertThat(bytesValues.valueAt(1).utf8ToString(), equalTo("3")); + assertTrue(bytesValues.advanceExact(2)); + assertThat(bytesValues.docValueCount(), equalTo(2)); + assertThat(bytesValues.nextValue().utf8ToString(), equalTo("1")); + assertThat(bytesValues.nextValue().utf8ToString(), equalTo("3")); - bytesValues.setDocument(3); - assertThat(bytesValues.count(), equalTo(1)); - assertThat(bytesValues.valueAt(0).utf8ToString(), equalTo("2")); + assertTrue(bytesValues.advanceExact(3)); + assertThat(bytesValues.docValueCount(), equalTo(1)); + assertThat(bytesValues.nextValue().utf8ToString(), equalTo("2")); - bytesValues.setDocument(4); - assertThat(bytesValues.count(), equalTo(2)); - assertThat(bytesValues.valueAt(0).utf8ToString(), equalTo("2")); - assertThat(bytesValues.valueAt(1).utf8ToString(), equalTo("4")); + assertTrue(bytesValues.advanceExact(4)); + assertThat(bytesValues.docValueCount(), equalTo(2)); + assertThat(bytesValues.nextValue().utf8ToString(), equalTo("2")); + assertThat(bytesValues.nextValue().utf8ToString(), equalTo("4")); - bytesValues.setDocument(5); - assertThat(bytesValues.count(), equalTo(2)); - assertThat(bytesValues.valueAt(0).utf8ToString(), equalTo("1")); - assertThat(bytesValues.valueAt(1).utf8ToString(), equalTo("5")); + assertTrue(bytesValues.advanceExact(5)); + assertThat(bytesValues.docValueCount(), equalTo(2)); + assertThat(bytesValues.nextValue().utf8ToString(), equalTo("1")); + assertThat(bytesValues.nextValue().utf8ToString(), equalTo("5")); - bytesValues.setDocument(6); - assertThat(bytesValues.count(), equalTo(1)); - assertThat(bytesValues.valueAt(0).utf8ToString(), equalTo("2")); + assertTrue(bytesValues.advanceExact(6)); + assertThat(bytesValues.docValueCount(), equalTo(1)); + assertThat(bytesValues.nextValue().utf8ToString(), equalTo("2")); - bytesValues.setDocument(7); - assertThat(bytesValues.count(), equalTo(0)); + assertFalse(bytesValues.advanceExact(7)); } } @@ -236,7 +235,7 @@ public void testThreads() throws Exception { ids[j] = BytesRef.deepCopyOf(id); } } - expected.put(context.reader().getCoreCacheKey(), ids); + expected.put(context.reader().getCoreCacheHelper().getKey(), ids); } for (int i = 0; i < numThreads; ++i) { @@ -249,7 +248,7 @@ public void run() { for (LeafReaderContext context : reader.leaves()) { AtomicParentChildFieldData leafData = global.load(context); SortedDocValues parentIds = leafData.getOrdinalsValues(parentType); - final BytesRef[] expectedIds = expected.get(context.reader().getCoreCacheKey()); + final BytesRef[] expectedIds = expected.get(context.reader().getCoreCacheHelper().getKey()); for (int j = 0; j < parentIds.getValueCount(); ++j) { final BytesRef id = parentIds.lookupOrd(j); assertEquals(expectedIds[j], id); diff --git a/core/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesDatesTests.java b/core/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesDatesTests.java index f8579efae737a..ed1b189e5f44b 100644 --- a/core/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesDatesTests.java +++ b/core/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesDatesTests.java @@ -19,7 +19,6 @@ package org.elasticsearch.index.fielddata; -import org.apache.lucene.index.SortedNumericDocValues; import org.elasticsearch.index.fielddata.ScriptDocValues.Dates; import org.elasticsearch.test.ESTestCase; import org.joda.time.DateTime; @@ -56,20 +55,23 @@ public void test() { } private Dates wrap(long[][] values) { - return new Dates(new SortedNumericDocValues() { + return new Dates(new AbstractSortedNumericDocValues() { long[] current; + int i; @Override - public void setDocument(int doc) { + public boolean advanceExact(int doc) { current = values[doc]; + i = 0; + return current.length > 0; } @Override - public int count() { + public int docValueCount() { return current.length; } @Override - public long valueAt(int index) { - return current[index]; + public long nextValue() { + return current[i++]; } }); } diff --git a/core/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesGeoPointsTests.java b/core/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesGeoPointsTests.java index 466ca0b6991a6..dfa19d102b981 100644 --- a/core/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesGeoPointsTests.java +++ b/core/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesGeoPointsTests.java @@ -30,22 +30,23 @@ public class ScriptDocValuesGeoPointsTests extends ESTestCase { private static MultiGeoPointValues wrap(final GeoPoint... points) { return new MultiGeoPointValues() { int docID = -1; + int i; @Override - public GeoPoint valueAt(int i) { + public GeoPoint nextValue() { if (docID != 0) { fail(); } - return points[i]; + return points[i++]; } @Override - public void setDocument(int docId) { - this.docID = docId; + public boolean advanceExact(int docId) { + return points.length > 0; } @Override - public int count() { + public int docValueCount() { if (docID != 0) { return 0; } diff --git a/core/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesLongsTests.java b/core/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesLongsTests.java index c6ec8ce749433..d1986be846347 100644 --- a/core/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesLongsTests.java +++ b/core/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesLongsTests.java @@ -19,7 +19,6 @@ package org.elasticsearch.index.fielddata; -import org.apache.lucene.index.SortedNumericDocValues; import org.elasticsearch.index.fielddata.ScriptDocValues.Longs; import org.elasticsearch.test.ESTestCase; import org.joda.time.DateTime; @@ -87,20 +86,22 @@ public void testDates() { } private Longs wrap(long[][] values) { - return new Longs(new SortedNumericDocValues() { + return new Longs(new AbstractSortedNumericDocValues() { long[] current; + int i; @Override - public void setDocument(int doc) { + public boolean advanceExact(int doc) { current = values[doc]; + return current.length > 0; } @Override - public int count() { + public int docValueCount() { return current.length; } @Override - public long valueAt(int index) { - return current[index]; + public long nextValue() { + return current[i++]; } }); } diff --git a/core/src/test/java/org/elasticsearch/index/fielddata/ordinals/MultiOrdinalsTests.java b/core/src/test/java/org/elasticsearch/index/fielddata/ordinals/MultiOrdinalsTests.java index c594427b7e83c..304940288d6a4 100644 --- a/core/src/test/java/org/elasticsearch/index/fielddata/ordinals/MultiOrdinalsTests.java +++ b/core/src/test/java/org/elasticsearch/index/fielddata/ordinals/MultiOrdinalsTests.java @@ -18,8 +18,8 @@ */ package org.elasticsearch.index.fielddata.ordinals; -import org.apache.lucene.index.RandomAccessOrds; import org.apache.lucene.index.SortedDocValues; +import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.util.packed.PackedInts; import org.elasticsearch.index.fielddata.FieldData; import org.elasticsearch.search.MultiValueMode; @@ -102,7 +102,7 @@ public int compare(OrdAndId o1, OrdAndId o2) { } }); Ordinals ords = creationMultiOrdinals(builder); - RandomAccessOrds docs = ords.ordinals(); + SortedSetDocValues docs = ords.ordinals(); final SortedDocValues singleOrds = MultiValueMode.MIN.select(docs); int docId = ordsAndIds.get(0).id; List docOrds = new ArrayList<>(); @@ -111,22 +111,18 @@ public int compare(OrdAndId o1, OrdAndId o2) { docOrds.add(ordAndId.ord); } else { if (!docOrds.isEmpty()) { - assertThat((long) singleOrds.getOrd(docId), equalTo(docOrds.get(0))); + assertTrue(singleOrds.advanceExact(docId)); + assertThat((long) singleOrds.ordValue(), equalTo(docOrds.get(0))); - docs.setDocument(docId); - final int numOrds = docs.cardinality(); - assertThat(numOrds, equalTo(docOrds.size())); - for (int i = 0; i < numOrds; i++) { - assertThat(docs.nextOrd(), equalTo(docOrds.get(i))); + assertTrue(docs.advanceExact(docId)); + for (Long ord : docOrds) { + assertThat(docs.nextOrd(), equalTo(ord)); } - final long[] array = new long[docOrds.size()]; - for (int i = 0; i < array.length; i++) { - array[i] = docOrds.get(i); - } - assertIter(docs, docId, array); + assertEquals(SortedSetDocValues.NO_MORE_ORDS, docs.nextOrd()); } for (int i = docId + 1; i < ordAndId.id; i++) { - assertThat((long) singleOrds.getOrd(i), equalTo(RandomAccessOrds.NO_MORE_ORDS)); + assertFalse(singleOrds.advanceExact(i)); + assertFalse(docs.advanceExact(i)); } docId = ordAndId.id; docOrds.clear(); @@ -209,18 +205,10 @@ public void testOrdinals() throws Exception { }; Ordinals ordinals = creationMultiOrdinals(builder); - RandomAccessOrds docs = ordinals.ordinals(); + SortedSetDocValues docs = ordinals.ordinals(); assertEquals(docs, ordinalPlan); } - protected static void assertIter(RandomAccessOrds docs, int docId, long... expectedOrdinals) { - docs.setDocument(docId); - assertThat(docs.cardinality(), equalTo(expectedOrdinals.length)); - for (long expectedOrdinal : expectedOrdinals) { - assertThat(docs.nextOrd(), equalTo(expectedOrdinal)); - } - } - public void testMultiValuesDocsWithOverlappingStorageArrays() throws Exception { int maxDoc = 7; long maxOrds = 15; @@ -261,11 +249,11 @@ public void testMultiValuesDocsWithOverlappingStorageArrays() throws Exception { }; Ordinals ordinals = new MultiOrdinals(builder, PackedInts.FASTEST); - RandomAccessOrds docs = ordinals.ordinals(); + SortedSetDocValues docs = ordinals.ordinals(); assertEquals(docs, ordinalPlan); } - private void assertEquals(RandomAccessOrds docs, long[][] ordinalPlan) { + private void assertEquals(SortedSetDocValues docs, long[][] ordinalPlan) throws IOException { long maxOrd = 0; for (int doc = 0; doc < ordinalPlan.length; ++doc) { if (ordinalPlan[doc].length > 0) { @@ -276,10 +264,12 @@ private void assertEquals(RandomAccessOrds docs, long[][] ordinalPlan) { assertThat(FieldData.isMultiValued(docs), equalTo(true)); for (int doc = 0; doc < ordinalPlan.length; ++doc) { long[] ords = ordinalPlan[doc]; - docs.setDocument(doc); - assertThat(docs.cardinality(), equalTo(ords.length)); - for (int i = 0; i < ords.length; ++i) { - assertThat(docs.ordAt(i), equalTo(ords[i])); + assertEquals(ords.length > 0, docs.advance(doc)); + if (ords.length > 0) { + for (long ord : ords) { + assertThat(docs.nextOrd(), equalTo(ord)); + } + assertThat(docs.nextOrd(), equalTo(SortedSetDocValues.NO_MORE_ORDS)); } } } diff --git a/core/src/test/java/org/elasticsearch/index/fielddata/ordinals/SingleOrdinalsTests.java b/core/src/test/java/org/elasticsearch/index/fielddata/ordinals/SingleOrdinalsTests.java index 9eb73e93324b7..b0fbda6a9407e 100644 --- a/core/src/test/java/org/elasticsearch/index/fielddata/ordinals/SingleOrdinalsTests.java +++ b/core/src/test/java/org/elasticsearch/index/fielddata/ordinals/SingleOrdinalsTests.java @@ -19,8 +19,8 @@ package org.elasticsearch.index.fielddata.ordinals; import org.apache.lucene.index.DocValues; -import org.apache.lucene.index.RandomAccessOrds; import org.apache.lucene.index.SortedDocValues; +import org.apache.lucene.index.SortedSetDocValues; import org.elasticsearch.test.ESTestCase; import java.io.IOException; @@ -48,12 +48,13 @@ public void testSvValues() throws IOException { Ordinals ords = builder.build(); assertThat(ords, instanceOf(SinglePackedOrdinals.class)); - RandomAccessOrds docs = ords.ordinals(); + SortedSetDocValues docs = ords.ordinals(); final SortedDocValues singleOrds = DocValues.unwrapSingleton(docs); assertNotNull(singleOrds); for (Map.Entry entry : controlDocToOrdinal.entrySet()) { - assertThat(entry.getValue(), equalTo((long) singleOrds.getOrd(entry.getKey()))); + assertTrue(singleOrds.advanceExact(entry.getKey())); + assertEquals(singleOrds.ordValue(), (long) entry.getValue()); } } diff --git a/core/src/test/java/org/elasticsearch/index/fielddata/plain/HalfFloatFielddataTests.java b/core/src/test/java/org/elasticsearch/index/fielddata/plain/HalfFloatFielddataTests.java index af15114eae768..19972e389b220 100644 --- a/core/src/test/java/org/elasticsearch/index/fielddata/plain/HalfFloatFielddataTests.java +++ b/core/src/test/java/org/elasticsearch/index/fielddata/plain/HalfFloatFielddataTests.java @@ -51,9 +51,9 @@ public void testSingleValued() throws IOException { SortedNumericDoubleValues values = new SortedNumericDVIndexFieldData.SortedNumericHalfFloatFieldData( reader, "half_float").getDoubleValues(); assertNotNull(FieldData.unwrapSingleton(values)); - values.setDocument(0); - assertEquals(1, values.count()); - assertEquals(3f, values.valueAt(0), 0f); + assertTrue(values.advanceExact(0)); + assertEquals(1, values.docValueCount()); + assertEquals(3f, values.nextValue(), 0f); IOUtils.close(dirReader, w, dir); } @@ -73,10 +73,10 @@ public void testMultiValued() throws IOException { SortedNumericDoubleValues values = new SortedNumericDVIndexFieldData.SortedNumericHalfFloatFieldData( reader, "half_float").getDoubleValues(); assertNull(FieldData.unwrapSingleton(values)); - values.setDocument(0); - assertEquals(2, values.count()); - assertEquals(2f, values.valueAt(0), 0f); - assertEquals(3f, values.valueAt(1), 0f); + assertTrue(values.advanceExact(0)); + assertEquals(2, values.docValueCount()); + assertEquals(2f, values.nextValue(), 0f); + assertEquals(3f, values.nextValue(), 0f); IOUtils.close(dirReader, w, dir); } } From 64e89e9b16bf6ae39b2230ce9bc1018f5b5918e4 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Thu, 6 Apr 2017 17:04:45 +0200 Subject: [PATCH 32/87] Remove irrelevant tests. --- .../highlight/PlainHighlighterTests.java | 53 ------------------- 1 file changed, 53 deletions(-) diff --git a/core/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/PlainHighlighterTests.java b/core/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/PlainHighlighterTests.java index b923c2464d831..18ba716f70cb2 100644 --- a/core/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/PlainHighlighterTests.java +++ b/core/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/PlainHighlighterTests.java @@ -19,29 +19,12 @@ package org.elasticsearch.search.fetch.subphase.highlight; -import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.MockAnalyzer; -import org.apache.lucene.analysis.standard.StandardAnalyzer; import org.apache.lucene.index.Term; -import org.apache.lucene.search.BooleanClause; -import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.PhraseQuery; import org.apache.lucene.search.Query; -import org.apache.lucene.search.TermQuery; -import org.apache.lucene.search.highlight.InvalidTokenOffsetsException; import org.apache.lucene.search.highlight.QueryScorer; -import org.apache.lucene.spatial.geopoint.search.GeoPointDistanceQuery; -import org.apache.lucene.spatial.geopoint.search.GeoPointInBBoxQuery; -import org.apache.lucene.spatial.geopoint.search.GeoPointInPolygonQuery; import org.apache.lucene.util.LuceneTestCase; -import org.elasticsearch.index.analysis.FieldNameAnalyzer; -import org.elasticsearch.search.fetch.subphase.highlight.CustomQueryScorer; - -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; - -import static org.hamcrest.Matchers.equalTo; public class PlainHighlighterTests extends LuceneTestCase { @@ -55,40 +38,4 @@ public void testHighlightPhrase() throws Exception { String[] frags = highlighter.getBestFragments(new MockAnalyzer(random()), "field", "bar foo bar foo", 10); assertArrayEquals(new String[] {"bar foo bar foo"}, frags); } - - public void checkGeoQueryHighlighting(Query geoQuery) throws IOException, InvalidTokenOffsetsException { - Map analysers = new HashMap(); - analysers.put("text", new StandardAnalyzer()); - FieldNameAnalyzer fieldNameAnalyzer = new FieldNameAnalyzer(analysers); - Query termQuery = new TermQuery(new Term("text", "failure")); - Query boolQuery = new BooleanQuery.Builder().add(new BooleanClause(geoQuery, BooleanClause.Occur.SHOULD)) - .add(new BooleanClause(termQuery, BooleanClause.Occur.SHOULD)).build(); - org.apache.lucene.search.highlight.Highlighter highlighter = - new org.apache.lucene.search.highlight.Highlighter(new CustomQueryScorer(boolQuery)); - String fragment = highlighter.getBestFragment(fieldNameAnalyzer.tokenStream("text", "Arbitrary text field which should not cause " + - "a failure"), "Arbitrary text field which should not cause a failure"); - assertThat(fragment, equalTo("Arbitrary text field which should not cause a failure")); - Query rewritten = boolQuery.rewrite(null); - highlighter = new org.apache.lucene.search.highlight.Highlighter(new CustomQueryScorer(rewritten)); - fragment = highlighter.getBestFragment(fieldNameAnalyzer.tokenStream("text", "Arbitrary text field which should not cause " + - "a failure"), "Arbitrary text field which should not cause a failure"); - assertThat(fragment, equalTo("Arbitrary text field which should not cause a failure")); - } - - public void testGeoPointInBBoxQueryHighlighting() throws IOException, InvalidTokenOffsetsException { - Query geoQuery = new GeoPointDistanceQuery("geo_point", -64.92354174306496, -170.15625, 5576757); - checkGeoQueryHighlighting(geoQuery); - } - - public void testGeoPointDistanceQueryHighlighting() throws IOException, InvalidTokenOffsetsException { - Query geoQuery = new GeoPointInBBoxQuery("geo_point", -64.92354174306496, 61.10078883158897, -170.15625, 118.47656249999999); - checkGeoQueryHighlighting(geoQuery); - } - - public void testGeoPointInPolygonQueryHighlighting() throws IOException, InvalidTokenOffsetsException { - double[] polyLats = new double[]{0, 60, 0, 0}; - double[] polyLons = new double[]{0, 60, 90, 0}; - Query geoQuery = new GeoPointInPolygonQuery("geo_point", polyLats, polyLons); - checkGeoQueryHighlighting(geoQuery); - } } From a67d19b8e7dcbf5fc39f63b717ecdb2e65ec7bf7 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Thu, 6 Apr 2017 17:07:09 +0200 Subject: [PATCH 33/87] Fix compile errors in ScriptValuesTests. --- .../support/ScriptValuesTests.java | 49 +++++++++++-------- 1 file changed, 29 insertions(+), 20 deletions(-) diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/support/ScriptValuesTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/support/ScriptValuesTests.java index bcb6d5318b75d..11e03a969d0e0 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/support/ScriptValuesTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/support/ScriptValuesTests.java @@ -28,6 +28,7 @@ import org.elasticsearch.search.aggregations.support.values.ScriptLongValues; import org.elasticsearch.test.ESTestCase; +import java.io.IOException; import java.util.Arrays; import java.util.Map; @@ -87,7 +88,7 @@ public double runAsDouble() { } - public void testLongs() { + public void testLongs() throws IOException { final Object[][] values = new Long[randomInt(10)][]; for (int i = 0; i < values.length; ++i) { Long[] longs = new Long[randomInt(8)]; @@ -100,15 +101,17 @@ public void testLongs() { FakeSearchScript script = new FakeSearchScript(values); ScriptLongValues scriptValues = new ScriptLongValues(script); for (int i = 0; i < values.length; ++i) { - scriptValues.setDocument(i); - assertEquals(values[i].length, scriptValues.docValueCount()); - for (int j = 0; j < values[i].length; ++j) { - assertEquals(values[i][j], scriptValues.nextValue()); + assertEquals(values[i].length > 0, scriptValues.advanceExact(i)); + if (values[i].length > 0) { + assertEquals(values[i].length, scriptValues.docValueCount()); + for (int j = 0; j < values[i].length; ++j) { + assertEquals(values[i][j], scriptValues.nextValue()); + } } } } - public void testBooleans() { + public void testBooleans() throws IOException { final Object[][] values = new Boolean[randomInt(10)][]; for (int i = 0; i < values.length; ++i) { Boolean[] booleans = new Boolean[randomInt(8)]; @@ -121,15 +124,17 @@ public void testBooleans() { FakeSearchScript script = new FakeSearchScript(values); ScriptLongValues scriptValues = new ScriptLongValues(script); for (int i = 0; i < values.length; ++i) { - scriptValues.setDocument(i); - assertEquals(values[i].length, scriptValues.docValueCount()); - for (int j = 0; j < values[i].length; ++j) { - assertEquals(values[i][j], scriptValues.nextValue() == 1L); + assertEquals(values[i].length > 0, scriptValues.advanceExact(i)); + if (values[i].length > 0) { + assertEquals(values[i].length, scriptValues.docValueCount()); + for (int j = 0; j < values[i].length; ++j) { + assertEquals(values[i][j], scriptValues.nextValue() == 1L); + } } } } - public void testDoubles() { + public void testDoubles() throws IOException { final Object[][] values = new Double[randomInt(10)][]; for (int i = 0; i < values.length; ++i) { Double[] doubles = new Double[randomInt(8)]; @@ -142,15 +147,17 @@ public void testDoubles() { FakeSearchScript script = new FakeSearchScript(values); ScriptDoubleValues scriptValues = new ScriptDoubleValues(script); for (int i = 0; i < values.length; ++i) { - scriptValues.setDocument(i); - assertEquals(values[i].length, scriptValues.docValueCount()); - for (int j = 0; j < values[i].length; ++j) { - assertEquals(values[i][j], scriptValues.nextValue()); + assertEquals(values[i].length > 0, scriptValues.advanceExact(i)); + if (values[i].length > 0) { + assertEquals(values[i].length, scriptValues.docValueCount()); + for (int j = 0; j < values[i].length; ++j) { + assertEquals(values[i][j], scriptValues.nextValue()); + } } } } - public void testBytes() { + public void testBytes() throws IOException { final String[][] values = new String[randomInt(10)][]; for (int i = 0; i < values.length; ++i) { String[] strings = new String[randomInt(8)]; @@ -163,10 +170,12 @@ public void testBytes() { FakeSearchScript script = new FakeSearchScript(values); ScriptBytesValues scriptValues = new ScriptBytesValues(script); for (int i = 0; i < values.length; ++i) { - scriptValues.setDocument(i); - assertEquals(values[i].length, scriptValues.docValueCount()); - for (int j = 0; j < values[i].length; ++j) { - assertEquals(new BytesRef(values[i][j]), scriptValues.nextValue()); + assertEquals(values[i].length > 0, scriptValues.advanceExact(i)); + if (values[i].length > 0) { + assertEquals(values[i].length, scriptValues.docValueCount()); + for (int j = 0; j < values[i].length; ++j) { + assertEquals(new BytesRef(values[i][j]), scriptValues.nextValue()); + } } } } From c9774e695ace3fdeb1771569b72b99ebf3329898 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Thu, 6 Apr 2017 17:26:13 +0200 Subject: [PATCH 34/87] Fix more compile errors. --- .../support/IncludeExcludeTests.java | 16 +- .../support/MissingValuesTests.java | 164 ++++++++++-------- 2 files changed, 94 insertions(+), 86 deletions(-) diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/support/IncludeExcludeTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/support/IncludeExcludeTests.java index 42f0df4beb9d3..c211f754d190a 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/support/IncludeExcludeTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/support/IncludeExcludeTests.java @@ -20,7 +20,6 @@ package org.elasticsearch.search.aggregations.support; import org.apache.lucene.index.DocValues; -import org.apache.lucene.index.RandomAccessOrds; import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.LongBitSet; @@ -30,6 +29,7 @@ import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.index.fielddata.AbstractSortedSetDocValues; import org.elasticsearch.index.query.QueryParseContext; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.aggregations.bucket.terms.support.IncludeExclude; @@ -58,13 +58,14 @@ public void testEmptyTermsWithOrds() throws IOException { } public void testSingleTermWithOrds() throws IOException { - RandomAccessOrds ords = new RandomAccessOrds() { + SortedSetDocValues ords = new AbstractSortedSetDocValues() { boolean consumed = true; @Override - public void setDocument(int docID) { + public boolean advanceExact(int docID) { consumed = false; + return true; } @Override @@ -88,15 +89,6 @@ public long getValueCount() { return 1; } - @Override - public long ordAt(int index) { - return 0; - } - - @Override - public int cardinality() { - return 1; - } }; IncludeExclude inexcl = new IncludeExclude( new TreeSet<>(Collections.singleton(new BytesRef("foo"))), diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/support/MissingValuesTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/support/MissingValuesTests.java index 2c819a671e55d..e29c59dd2328b 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/support/MissingValuesTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/support/MissingValuesTests.java @@ -21,24 +21,27 @@ import com.carrotsearch.randomizedtesting.generators.RandomPicks; import com.carrotsearch.randomizedtesting.generators.RandomStrings; -import org.apache.lucene.index.RandomAccessOrds; + import org.apache.lucene.index.SortedNumericDocValues; +import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.TestUtil; import org.elasticsearch.common.geo.GeoPoint; -import org.elasticsearch.index.fielddata.AbstractRandomAccessOrds; +import org.elasticsearch.index.fielddata.AbstractSortedNumericDocValues; +import org.elasticsearch.index.fielddata.AbstractSortedSetDocValues; import org.elasticsearch.index.fielddata.MultiGeoPointValues; import org.elasticsearch.index.fielddata.SortedBinaryDocValues; import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; import org.elasticsearch.test.ESTestCase; +import java.io.IOException; import java.util.Arrays; import java.util.HashSet; import java.util.Set; public class MissingValuesTests extends ESTestCase { - public void testMissingBytes() { + public void testMissingBytes() throws IOException { final int numDocs = TestUtil.nextInt(random(), 1, 100); final BytesRef[][] values = new BytesRef[numDocs][]; for (int i = 0; i < numDocs; ++i) { @@ -50,40 +53,43 @@ public void testMissingBytes() { } SortedBinaryDocValues asBinaryValues = new SortedBinaryDocValues() { - int i = -1; + int doc = -1; + int i; @Override - public BytesRef valueAt(int index) { - return values[i][index]; + public BytesRef nextValue() { + return values[doc][doc++]; } @Override - public void setDocument(int docId) { - i = docId; + public boolean advanceExact(int docId) { + doc = docId; + doc = 0; + return values[doc].length > 0; } @Override - public int count() { - return values[i].length; + public int docValueCount() { + return values[doc].length; } }; final BytesRef missing = new BytesRef(RandomStrings.randomAsciiOfLength(random(), 2)); SortedBinaryDocValues withMissingReplaced = MissingValues.replaceMissing(asBinaryValues, missing); for (int i = 0; i < numDocs; ++i) { - withMissingReplaced.setDocument(i); + assertTrue(withMissingReplaced.advanceExact(i)); if (values[i].length > 0) { - assertEquals(values[i].length, withMissingReplaced.count()); + assertEquals(values[i].length, withMissingReplaced.docValueCount()); for (int j = 0; j < values[i].length; ++j) { - assertEquals(values[i][j], withMissingReplaced.valueAt(j)); + assertEquals(values[i][j], withMissingReplaced.nextValue()); } } else { - assertEquals(1, withMissingReplaced.count()); - assertEquals(missing, withMissingReplaced.valueAt(0)); + assertEquals(1, withMissingReplaced.docValueCount()); + assertEquals(missing, withMissingReplaced.nextValue()); } } } - public void testMissingOrds() { + public void testMissingOrds() throws IOException { final int numDocs = TestUtil.nextInt(random(), 1, 100); final int numOrds = TestUtil.nextInt(random(), 1, 10); @@ -105,13 +111,16 @@ public void testMissingOrds() { ords[i][j] = TestUtil.nextInt(random(), ords[i][j], maxOrd - 1); } } - RandomAccessOrds asRandomAccessOrds = new AbstractRandomAccessOrds() { + SortedSetDocValues asRandomAccessOrds = new AbstractSortedSetDocValues() { - int i = -1; + int doc = -1; + int i; @Override - public void doSetDocument(int docID) { - i = docID; + public boolean advanceExact(int docID) { + doc = docID; + doc = 0; + return values[doc].length > 0; } @Override @@ -125,13 +134,12 @@ public long getValueCount() { } @Override - public long ordAt(int index) { - return ords[i][index]; - } - - @Override - public int cardinality() { - return ords[i].length; + public long nextOrd() { + if (doc < ords[doc].length) { + return ords[doc][doc++]; + } else { + return NO_MORE_ORDS; + } } }; @@ -139,28 +147,28 @@ public int cardinality() { final BytesRef missingMissing = new BytesRef(RandomStrings.randomAsciiOfLength(random(), 5)); for (BytesRef missing : Arrays.asList(existingMissing, missingMissing)) { - RandomAccessOrds withMissingReplaced = MissingValues.replaceMissing(asRandomAccessOrds, missing); + SortedSetDocValues withMissingReplaced = MissingValues.replaceMissing(asRandomAccessOrds, missing); if (valueSet.contains(missing)) { assertEquals(values.length, withMissingReplaced.getValueCount()); } else { assertEquals(values.length + 1, withMissingReplaced.getValueCount()); } for (int i = 0; i < numDocs; ++i) { - withMissingReplaced.setDocument(i); + assertTrue(withMissingReplaced.advanceExact(i)); if (ords[i].length > 0) { - assertEquals(ords[i].length, withMissingReplaced.cardinality()); - for (int j = 0; j < ords[i].length; ++j) { - assertEquals(values[ords[i][j]], withMissingReplaced.lookupOrd(withMissingReplaced.ordAt(j))); + for (long ord : ords[i]) { + assertEquals(ord, withMissingReplaced.nextOrd()); } + assertEquals(SortedSetDocValues.NO_MORE_ORDS, withMissingReplaced.nextOrd()); } else { - assertEquals(1, withMissingReplaced.cardinality()); - assertEquals(missing, withMissingReplaced.lookupOrd(withMissingReplaced.ordAt(0))); + assertEquals(missing, withMissingReplaced.lookupOrd(withMissingReplaced.nextOrd())); + assertEquals(SortedSetDocValues.NO_MORE_ORDS, withMissingReplaced.nextOrd()); } } } } - public void testMissingLongs() { + public void testMissingLongs() throws IOException { final int numDocs = TestUtil.nextInt(random(), 1, 100); final int[][] values = new int[numDocs][]; for (int i = 0; i < numDocs; ++i) { @@ -170,42 +178,45 @@ public void testMissingLongs() { } Arrays.sort(values[i]); } - SortedNumericDocValues asNumericValues = new SortedNumericDocValues() { + SortedNumericDocValues asNumericValues = new AbstractSortedNumericDocValues() { - int i = -1; + int doc = -1; + int i; @Override - public long valueAt(int index) { - return values[i][index]; + public long nextValue() { + return values[doc][doc++]; } @Override - public void setDocument(int docId) { - i = docId; + public boolean advanceExact(int docId) { + doc = docId; + doc = 0; + return values[doc].length > 0; } @Override - public int count() { - return values[i].length; + public int docValueCount() { + return values[doc].length; } }; final long missing = randomInt(); SortedNumericDocValues withMissingReplaced = MissingValues.replaceMissing(asNumericValues, missing); for (int i = 0; i < numDocs; ++i) { - withMissingReplaced.setDocument(i); + assertTrue(withMissingReplaced.advanceExact(i)); if (values[i].length > 0) { - assertEquals(values[i].length, withMissingReplaced.count()); + assertEquals(values[i].length, withMissingReplaced.docValueCount()); for (int j = 0; j < values[i].length; ++j) { - assertEquals(values[i][j], withMissingReplaced.valueAt(j)); + assertEquals(values[i][j], withMissingReplaced.nextValue()); } } else { - assertEquals(1, withMissingReplaced.count()); - assertEquals(missing, withMissingReplaced.valueAt(0)); + assertEquals(1, withMissingReplaced.docValueCount()); + assertEquals(missing, withMissingReplaced.nextValue()); } } } - public void testMissingDoubles() { + public void testMissingDoubles() throws IOException { final int numDocs = TestUtil.nextInt(random(), 1, 100); final double[][] values = new double[numDocs][]; for (int i = 0; i < numDocs; ++i) { @@ -217,40 +228,42 @@ public void testMissingDoubles() { } SortedNumericDoubleValues asNumericValues = new SortedNumericDoubleValues() { - int i = -1; + int doc = -1; + int i; @Override - public double valueAt(int index) { - return values[i][index]; + public double nextValue() { + return values[doc][doc++]; } @Override - public void setDocument(int docId) { - i = docId; + public boolean advanceExact(int docId) { + doc = docId; + return true; } @Override - public int count() { - return values[i].length; + public int docValueCount() { + return values[doc].length; } }; final long missing = randomInt(); SortedNumericDoubleValues withMissingReplaced = MissingValues.replaceMissing(asNumericValues, missing); for (int i = 0; i < numDocs; ++i) { - withMissingReplaced.setDocument(i); + assertTrue(withMissingReplaced.advanceExact(i)); if (values[i].length > 0) { - assertEquals(values[i].length, withMissingReplaced.count()); + assertEquals(values[i].length, withMissingReplaced.docValueCount()); for (int j = 0; j < values[i].length; ++j) { - assertEquals(values[i][j], withMissingReplaced.valueAt(j), 0); + assertEquals(values[i][j], withMissingReplaced.nextValue(), 0); } } else { - assertEquals(1, withMissingReplaced.count()); - assertEquals(missing, withMissingReplaced.valueAt(0), 0); + assertEquals(1, withMissingReplaced.docValueCount()); + assertEquals(missing, withMissingReplaced.nextValue(), 0); } } } - public void testMissingGeoPoints() { + public void testMissingGeoPoints() throws IOException { final int numDocs = TestUtil.nextInt(random(), 1, 100); final GeoPoint[][] values = new GeoPoint[numDocs][]; for (int i = 0; i < numDocs; ++i) { @@ -261,35 +274,38 @@ public void testMissingGeoPoints() { } MultiGeoPointValues asGeoValues = new MultiGeoPointValues() { - int i = -1; + int doc = -1; + int i; @Override - public GeoPoint valueAt(int index) { - return values[i][index]; + public GeoPoint nextValue() { + return values[doc][i++]; } @Override - public void setDocument(int docId) { - i = docId; + public boolean advanceExact(int docId) { + doc = docId; + i = 0; + return values[doc].length > 0; } @Override - public int count() { - return values[i].length; + public int docValueCount() { + return values[doc].length; } }; final GeoPoint missing = new GeoPoint(randomDouble() * 90, randomDouble() * 180); MultiGeoPointValues withMissingReplaced = MissingValues.replaceMissing(asGeoValues, missing); for (int i = 0; i < numDocs; ++i) { - withMissingReplaced.setDocument(i); + assertTrue(withMissingReplaced.advanceExact(i)); if (values[i].length > 0) { - assertEquals(values[i].length, withMissingReplaced.count()); + assertEquals(values[i].length, withMissingReplaced.docValueCount()); for (int j = 0; j < values[i].length; ++j) { - assertEquals(values[i][j], withMissingReplaced.valueAt(j)); + assertEquals(values[i][j], withMissingReplaced.nextValue()); } } else { - assertEquals(1, withMissingReplaced.count()); - assertEquals(missing, withMissingReplaced.valueAt(0)); + assertEquals(1, withMissingReplaced.docValueCount()); + assertEquals(missing, withMissingReplaced.nextValue()); } } } From 8fad4a07c6dc1b7628a9edccd1a0a153aa1eb42b Mon Sep 17 00:00:00 2001 From: Colin Goodheart-Smithe Date: Fri, 7 Apr 2017 10:27:47 +0000 Subject: [PATCH 35/87] Fixes compile errors in ScriptDovValues --- .../index/fielddata/ScriptDocValues.java | 310 +++++++++++++----- 1 file changed, 224 insertions(+), 86 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java b/core/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java index bfa23eb8f04be..ce038298ce0c7 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java @@ -21,6 +21,7 @@ import org.apache.lucene.index.SortedNumericDocValues; +import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.geo.GeoHashUtils; import org.elasticsearch.common.geo.GeoPoint; @@ -32,6 +33,7 @@ import org.joda.time.MutableDateTime; import org.joda.time.ReadableDateTime; +import java.io.IOException; import java.util.AbstractList; import java.util.Comparator; import java.util.List; @@ -46,7 +48,7 @@ public abstract class ScriptDocValues extends AbstractList { /** * Set the current doc ID. */ - public abstract void setNextDocId(int docId); + public abstract void setNextDocId(int docId) throws IOException; /** * Return a copy of the list of the values for the current document. @@ -83,24 +85,46 @@ public final void sort(Comparator c) { public static final class Strings extends ScriptDocValues { - private final SortedBinaryDocValues values; + private final SortedBinaryDocValues in; + private BytesRef[] values; + private int count; - public Strings(SortedBinaryDocValues values) { - this.values = values; + public Strings(SortedBinaryDocValues in) { + this.in = in; } @Override - public void setNextDocId(int docId) { - values.setDocument(docId); + public void setNextDocId(int docId) throws IOException { + if (in.advanceExact(docId)) { + resize(in.docValueCount()); + for (int i = 0; i < count; i++) { + values[i] = in.nextValue(); + } + } else { + resize(0); + } + } + + /** + * Set the {@link #size()} and ensure that the {@link #values} array can + * store at least that many entries. + */ + protected final void resize(int newSize) { + count = newSize; + if (values == null) { + values = new BytesRef[newSize]; + } else { + values = ArrayUtil.grow(values, count); + } } public SortedBinaryDocValues getInternalValues() { - return this.values; + return this.in; } public BytesRef getBytesValue() { - if (values.count() > 0) { - return values.valueAt(0); + if (size() > 0) { + return values[0]; } else { return null; } @@ -117,12 +141,12 @@ public String getValue() { @Override public String get(int index) { - return values.valueAt(index).utf8ToString(); + return values[index].utf8ToString(); } @Override public int size() { - return values.count(); + return count; } } @@ -130,48 +154,70 @@ public int size() { public static final class Longs extends ScriptDocValues { protected static final DeprecationLogger deprecationLogger = new DeprecationLogger(ESLoggerFactory.getLogger(Longs.class)); - private final SortedNumericDocValues values; + private final SortedNumericDocValues in; + private Long[] values; + private int count; private Dates dates; - public Longs(SortedNumericDocValues values) { - this.values = values; + public Longs(SortedNumericDocValues in) { + this.in = in; + } @Override - public void setNextDocId(int docId) { - values.setDocument(docId); + public void setNextDocId(int docId) throws IOException { + if (in.advanceExact(docId)) { + resize(in.docValueCount()); + for (int i = 0; i < count; i++) { + values[i] = in.nextValue(); + } + } else { + resize(0); + } if (dates != null) { dates.refreshArray(); } } + /** + * Set the {@link #size()} and ensure that the {@link #values} array can + * store at least that many entries. + */ + protected final void resize(int newSize) { + count = newSize; + if (values == null) { + values = new Long[newSize]; + } else { + values = ArrayUtil.grow(values, count); + } + } + public SortedNumericDocValues getInternalValues() { - return this.values; + return this.in; } public long getValue() { - int numValues = values.count(); - if (numValues == 0) { + if (count == 0) { return 0L; } - return values.valueAt(0); + return values[0]; } @Deprecated - public ReadableDateTime getDate() { + public ReadableDateTime getDate() throws IOException { deprecationLogger.deprecated("getDate on numeric fields is deprecated. Use a date field to get dates."); if (dates == null) { - dates = new Dates(values); + dates = new Dates(in); dates.refreshArray(); } return dates.getValue(); } @Deprecated - public List getDates() { + public List getDates() throws IOException { deprecationLogger.deprecated("getDates on numeric fields is deprecated. Use a date field to get dates."); if (dates == null) { - dates = new Dates(values); + dates = new Dates(in); dates.refreshArray(); } return dates; @@ -179,12 +225,12 @@ public List getDates() { @Override public Long get(int index) { - return values.valueAt(index); + return values[index]; } @Override public int size() { - return values.count(); + return count; } } @@ -193,22 +239,24 @@ public static final class Dates extends ScriptDocValues { private static final ReadableDateTime EPOCH = new DateTime(0, DateTimeZone.UTC); - private final SortedNumericDocValues values; + private final SortedNumericDocValues in; /** * Values wrapped in {@link MutableDateTime}. Null by default an allocated on first usage so we allocate a reasonably size. We keep * this array so we don't have allocate new {@link MutableDateTime}s on every usage. Instead we reuse them for every document. */ private MutableDateTime[] dates; + private int count; - public Dates(SortedNumericDocValues values) { - this.values = values; + public Dates(SortedNumericDocValues in) { + this.in = in; } /** - * Fetch the first field value or 0 millis after epoch if there are no values. + * Fetch the first field value or 0 millis after epoch if there are no + * in. */ public ReadableDateTime getValue() { - if (values.count() == 0) { + if (count == 0) { return EPOCH; } return get(0); @@ -234,113 +282,160 @@ public List getDates() { @Override public ReadableDateTime get(int index) { - if (index >= values.count()) { + if (index >= count) { throw new IndexOutOfBoundsException( - "attempted to fetch the [" + index + "] date when there are only [" + values.count() + "] dates."); + "attempted to fetch the [" + index + "] date when there are only [" + + count + "] dates."); } return dates[index]; } @Override public int size() { - return values.count(); + return count; } @Override - public void setNextDocId(int docId) { - values.setDocument(docId); + public void setNextDocId(int docId) throws IOException { + if (in.advanceExact(docId)) { + count = in.docValueCount(); + } else { + count = 0; + } refreshArray(); } /** * Refresh the backing array. Package private so it can be called when {@link Longs} loads dates. */ - void refreshArray() { - if (values.count() == 0) { + void refreshArray() throws IOException { + if (count == 0) { return; } if (dates == null) { // Happens for the document. We delay allocating dates so we can allocate it with a reasonable size. - dates = new MutableDateTime[values.count()]; + dates = new MutableDateTime[count]; for (int i = 0; i < dates.length; i++) { - dates[i] = new MutableDateTime(values.valueAt(i), DateTimeZone.UTC); + dates[i] = new MutableDateTime(in.nextValue(), DateTimeZone.UTC); } return; } - if (values.count() > dates.length) { + if (count > dates.length) { // Happens when we move to a new document and it has more dates than any documents before it. MutableDateTime[] backup = dates; - dates = new MutableDateTime[values.count()]; + dates = new MutableDateTime[count]; System.arraycopy(backup, 0, dates, 0, backup.length); for (int i = 0; i < backup.length; i++) { - dates[i].setMillis(values.valueAt(i)); + dates[i].setMillis(in.nextValue()); } for (int i = backup.length; i < dates.length; i++) { - dates[i] = new MutableDateTime(values.valueAt(i), DateTimeZone.UTC); + dates[i] = new MutableDateTime(in.nextValue(), DateTimeZone.UTC); } return; } - for (int i = 0; i < values.count(); i++) { - dates[i].setMillis(values.valueAt(i)); + for (int i = 0; i < count; i++) { + dates[i].setMillis(in.nextValue()); } } } public static final class Doubles extends ScriptDocValues { - private final SortedNumericDoubleValues values; + private final SortedNumericDoubleValues in; + private Double[] values; + private int count; - public Doubles(SortedNumericDoubleValues values) { - this.values = values; + public Doubles(SortedNumericDoubleValues in) { + this.in = in; } @Override - public void setNextDocId(int docId) { - values.setDocument(docId); + public void setNextDocId(int docId) throws IOException { + if (in.advanceExact(docId)) { + resize(in.docValueCount()); + for (int i = 0; i < count; i++) { + values[i] = in.nextValue(); + } + } else { + resize(0); + } + } + + /** + * Set the {@link #size()} and ensure that the {@link #values} array can + * store at least that many entries. + */ + protected final void resize(int newSize) { + count = newSize; + if (values == null) { + values = new Double[newSize]; + } else { + values = ArrayUtil.grow(values, count); + } } public SortedNumericDoubleValues getInternalValues() { - return this.values; + return this.in; } public double getValue() { - int numValues = values.count(); - if (numValues == 0) { + if (count == 0) { return 0d; } - return values.valueAt(0); + return values[0]; } @Override public Double get(int index) { - return values.valueAt(index); + return values[index]; } @Override public int size() { - return values.count(); + return count; } } public static final class GeoPoints extends ScriptDocValues { - private final MultiGeoPointValues values; + private final MultiGeoPointValues in; + private GeoPoint[] values; + private int count; - public GeoPoints(MultiGeoPointValues values) { - this.values = values; + public GeoPoints(MultiGeoPointValues in) { + this.in = in; } @Override - public void setNextDocId(int docId) { - values.setDocument(docId); + public void setNextDocId(int docId) throws IOException { + if (in.advanceExact(docId)) { + resize(in.docValueCount()); + for (int i = 0; i < count; i++) { + values[i] = in.nextValue(); + } + } else { + resize(0); + } + } + + /** + * Set the {@link #size()} and ensure that the {@link #values} array can + * store at least that many entries. + */ + protected final void resize(int newSize) { + count = newSize; + if (values == null) { + values = new GeoPoint[newSize]; + } else { + values = ArrayUtil.grow(values, count); + } } public GeoPoint getValue() { - int numValues = values.count(); - if (numValues == 0) { + if (count == 0) { return null; } - return values.valueAt(0); + return values[0]; } public double getLat() { @@ -371,13 +466,13 @@ public double getLon() { @Override public GeoPoint get(int index) { - final GeoPoint point = values.valueAt(index); + final GeoPoint point = values[index]; return new GeoPoint(point.lat(), point.lon()); } @Override public int size() { - return values.count(); + return count; } public double arcDistance(double lat, double lon) { @@ -420,66 +515,109 @@ public double geohashDistanceWithDefault(String geohash, double defaultValue) { public static final class Booleans extends ScriptDocValues { - private final SortedNumericDocValues values; + private final SortedNumericDocValues in; + private Boolean[] values; + private int count; - public Booleans(SortedNumericDocValues values) { - this.values = values; + public Booleans(SortedNumericDocValues in) { + this.in = in; } @Override - public void setNextDocId(int docId) { - values.setDocument(docId); + public void setNextDocId(int docId) throws IOException { + if (in.advanceExact(docId)) { + resize(in.docValueCount()); + for (int i = 0; i < count; i++) { + values[i] = in.nextValue() == 1; + } + } else { + resize(0); + } + } + + /** + * Set the {@link #size()} and ensure that the {@link #values} array can + * store at least that many entries. + */ + protected final void resize(int newSize) { + count = newSize; + if (values == null) { + values = new Boolean[newSize]; + } else { + values = ArrayUtil.grow(values, count); + } } public boolean getValue() { - return values.count() != 0 && values.valueAt(0) == 1; + return count != 0 && values[0]; } @Override public Boolean get(int index) { - return values.valueAt(index) == 1; + return values[index]; } @Override public int size() { - return values.count(); + return count; } } public static final class BytesRefs extends ScriptDocValues { - private final SortedBinaryDocValues values; + private final SortedBinaryDocValues in; + private BytesRef[] values; + private int count; - public BytesRefs(SortedBinaryDocValues values) { - this.values = values; + public BytesRefs(SortedBinaryDocValues in) { + this.in = in; } @Override - public void setNextDocId(int docId) { - values.setDocument(docId); + public void setNextDocId(int docId) throws IOException { + if (in.advanceExact(docId)) { + resize(in.docValueCount()); + for (int i = 0; i < count; i++) { + values[i] = in.nextValue(); + } + } else { + resize(0); + } + } + + /** + * Set the {@link #size()} and ensure that the {@link #values} array can + * store at least that many entries. + */ + protected final void resize(int newSize) { + count = newSize; + if (values == null) { + values = new BytesRef[newSize]; + } else { + values = ArrayUtil.grow(values, count); + } } public SortedBinaryDocValues getInternalValues() { - return this.values; + return this.in; } public BytesRef getValue() { - int numValues = values.count(); - if (numValues == 0) { + if (count == 0) { return new BytesRef(); } - return values.valueAt(0); + return values[0]; } @Override public BytesRef get(int index) { - return values.valueAt(index); + return values[index]; } @Override public int size() { - return values.count(); + return count; } } } From 42fe06218ef5c3d15afb73ebbec84c7cde4c4506 Mon Sep 17 00:00:00 2001 From: Jim Ferenczi Date: Fri, 7 Apr 2017 12:43:06 +0200 Subject: [PATCH 36/87] Remove last instance of disable coord and adds a TODO regarding the breaking change that this removal implies in the handling of minimum_should_match --- .../elasticsearch/common/lucene/search/Queries.java | 5 +---- .../index/query/SimpleQueryStringBuilderTests.java | 11 +++++++++-- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/common/lucene/search/Queries.java b/core/src/main/java/org/elasticsearch/common/lucene/search/Queries.java index a7a94cf206442..acf3f9ffdf8a3 100644 --- a/core/src/main/java/org/elasticsearch/common/lucene/search/Queries.java +++ b/core/src/main/java/org/elasticsearch/common/lucene/search/Queries.java @@ -168,10 +168,7 @@ public static Query applyMinimumShouldMatch(BooleanQuery query, @Nullable String * otherwise return the original query. */ public static Query maybeApplyMinimumShouldMatch(Query query, @Nullable String minimumShouldMatch) { - // If the coordination factor is disabled on a boolean query we don't apply the minimum should match. - // This is done to make sure that the minimum_should_match doesn't get applied when there is only one word - // and multiple variations of the same word in the query (synonyms for instance). - if (query instanceof BooleanQuery && !((BooleanQuery) query).isCoordDisabled()) { + if (query instanceof BooleanQuery) { return applyMinimumShouldMatch((BooleanQuery) query, minimumShouldMatch); } else if (query instanceof ExtendedCommonTermsQuery) { ((ExtendedCommonTermsQuery)query).setLowFreqMinimumNumberShouldMatch(minimumShouldMatch); diff --git a/core/src/test/java/org/elasticsearch/index/query/SimpleQueryStringBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/SimpleQueryStringBuilderTests.java index 5e221748e2bb7..aa515efd0020f 100644 --- a/core/src/test/java/org/elasticsearch/index/query/SimpleQueryStringBuilderTests.java +++ b/core/src/test/java/org/elasticsearch/index/query/SimpleQueryStringBuilderTests.java @@ -244,9 +244,16 @@ protected void doAssertLuceneQuery(SimpleQueryStringBuilder queryBuilder, Query Map.Entry field = fieldsIterator.next(); assertTermOrBoostQuery(booleanClause.getQuery(), field.getKey(), queryBuilder.value(), field.getValue()); } - if (queryBuilder.minimumShouldMatch() != null && !boolQuery.isCoordDisabled()) { + /** + * TODO: + * Test disabled because we cannot check min should match consistently: + * https://github.com/elastic/elasticsearch/issues/23966 + * + if (queryBuilder.minimumShouldMatch() != null && !boolQuery.isCoordDisabled()) { assertThat(boolQuery.getMinimumNumberShouldMatch(), greaterThan(0)); - } + } + * + **/ } else if (queryBuilder.fields().size() == 1) { Map.Entry field = queryBuilder.fields().entrySet().iterator().next(); assertTermOrBoostQuery(query, field.getKey(), queryBuilder.value(), field.getValue()); From badf9cbdb309011788628fa89b8d71bdf06c4e9c Mon Sep 17 00:00:00 2001 From: Jim Ferenczi Date: Fri, 7 Apr 2017 15:29:57 +0200 Subject: [PATCH 37/87] Fix compile error due to API change --- .../search/suggest/completion/CompletionSuggester.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggester.java b/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggester.java index abc8d209cd393..0b127b2eeef7d 100644 --- a/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggester.java +++ b/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggester.java @@ -83,7 +83,7 @@ protected Suggest.Suggestion scoreDocMap; // TODO: expose dup removal - + TopDocumentsCollector(int num) { super(1, false); // TODO hack, we don't use the underlying pq, so we allocate a size of 1 this.num = num; From 1110b7c1882f6e4631aaa072bbd38f16df4e5332 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Mon, 10 Apr 2017 10:58:39 +0200 Subject: [PATCH 38/87] Fix some geo compile errors. --- .../common/geo/GeoHashUtils.java | 48 ++++++++++++++++--- .../elasticsearch/common/geo/GeoPoint.java | 7 ++- .../elasticsearch/common/geo/GeoUtils.java | 40 +++++++++------- 3 files changed, 66 insertions(+), 29 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/common/geo/GeoHashUtils.java b/core/src/main/java/org/elasticsearch/common/geo/GeoHashUtils.java index 0f1ddcd376ce3..edac846bc5e43 100644 --- a/core/src/main/java/org/elasticsearch/common/geo/GeoHashUtils.java +++ b/core/src/main/java/org/elasticsearch/common/geo/GeoHashUtils.java @@ -21,8 +21,8 @@ import java.util.ArrayList; import java.util.Collection; -import org.apache.lucene.spatial.geopoint.document.GeoPointField; import org.apache.lucene.geo.Rectangle; +import org.apache.lucene.spatial.util.MortonEncoder; import org.apache.lucene.util.BitUtil; /** @@ -42,19 +42,35 @@ public class GeoHashUtils { /** maximum precision for geohash strings */ public static final int PRECISION = 12; - private static final short MORTON_OFFSET = (GeoPointField.BITS<<1) - (PRECISION*5); + /** number of bits used for quantizing latitude and longitude values */ + public static final short BITS = 31; + /** scaling factors to convert lat/lon into unsigned space */ + private static final double LAT_SCALE = (0x1L<>> 2; + } + /** * Encode lon/lat to the geohash based long format (lon/lat interleaved, 4 least significant bits = level) */ public static final long longEncode(final double lon, final double lat, final int level) { // shift to appropriate level final short msf = (short)(((12 - level) * 5) + MORTON_OFFSET); - return ((BitUtil.flipFlop(GeoPointField.encodeLatLon(lat, lon)) >>> msf) << 4) | level; + return ((BitUtil.flipFlop(encodeLatLon(lat, lon)) >>> msf) << 4) | level; } /** @@ -120,7 +136,7 @@ public static final String stringEncode(final double lon, final double lat) { */ public static final String stringEncode(final double lon, final double lat, final int level) { // convert to geohashlong - final long ghLong = fromMorton(GeoPointField.encodeLatLon(lat, lon), level); + final long ghLong = fromMorton(encodeLatLon(lat, lon), level); return stringEncode(ghLong); } @@ -141,7 +157,7 @@ public static final String stringEncodeFromMortonLong(long hashedVal, final int StringBuilder geoHash = new StringBuilder(); short precision = 0; - final short msf = (GeoPointField.BITS<<1)-5; + final short msf = (BITS<<1)-5; long mask = 31L<>>(msf-(precision*5)))]); @@ -303,13 +319,31 @@ public static final > E addNeighbors(String return neighbors; } + /** decode longitude value from morton encoded geo point */ + public static final double decodeLongitude(final long hash) { + return unscaleLon(BitUtil.deinterleave(hash)); + } + + /** decode latitude value from morton encoded geo point */ + public static final double decodeLatitude(final long hash) { + return unscaleLat(BitUtil.deinterleave(hash >>> 1)); + } + + private static final double unscaleLon(final long val) { + return (val / LON_SCALE) - 180; + } + + private static final double unscaleLat(final long val) { + return (val / LAT_SCALE) - 90; + } + /** returns the latitude value from the string based geohash */ public static final double decodeLatitude(final String geohash) { - return GeoPointField.decodeLatitude(mortonEncode(geohash)); + return decodeLatitude(mortonEncode(geohash)); } /** returns the latitude value from the string based geohash */ public static final double decodeLongitude(final String geohash) { - return GeoPointField.decodeLongitude(mortonEncode(geohash)); + return decodeLongitude(mortonEncode(geohash)); } } diff --git a/core/src/main/java/org/elasticsearch/common/geo/GeoPoint.java b/core/src/main/java/org/elasticsearch/common/geo/GeoPoint.java index f76720b9ed6b2..125bc5aefcf9f 100644 --- a/core/src/main/java/org/elasticsearch/common/geo/GeoPoint.java +++ b/core/src/main/java/org/elasticsearch/common/geo/GeoPoint.java @@ -23,7 +23,6 @@ import org.apache.lucene.document.LatLonPoint; import org.apache.lucene.geo.GeoEncodingUtils; import org.apache.lucene.index.IndexableField; -import org.apache.lucene.spatial.geopoint.document.GeoPointField; import org.apache.lucene.util.BitUtil; import org.apache.lucene.util.BytesRef; @@ -87,8 +86,8 @@ public GeoPoint resetFromString(String value) { } public GeoPoint resetFromIndexHash(long hash) { - lon = GeoPointField.decodeLongitude(hash); - lat = GeoPointField.decodeLatitude(hash); + lon = GeoHashUtils.decodeLongitude(hash); + lat = GeoHashUtils.decodeLatitude(hash); return this; } @@ -112,7 +111,7 @@ public GeoPoint resetFromIndexableField(IndexableField field) { public GeoPoint resetFromGeoHash(String geohash) { final long hash = mortonEncode(geohash); - return this.reset(GeoPointField.decodeLatitude(hash), GeoPointField.decodeLongitude(hash)); + return this.reset(GeoHashUtils.decodeLatitude(hash), GeoHashUtils.decodeLongitude(hash)); } public GeoPoint resetFromGeoHash(long geohashLong) { diff --git a/core/src/main/java/org/elasticsearch/common/geo/GeoUtils.java b/core/src/main/java/org/elasticsearch/common/geo/GeoUtils.java index 81215906a06ae..b1448796821a4 100644 --- a/core/src/main/java/org/elasticsearch/common/geo/GeoUtils.java +++ b/core/src/main/java/org/elasticsearch/common/geo/GeoUtils.java @@ -22,7 +22,6 @@ import org.apache.lucene.geo.Rectangle; import org.apache.lucene.spatial.prefix.tree.GeohashPrefixTree; import org.apache.lucene.spatial.prefix.tree.QuadPrefixTree; -import org.apache.lucene.util.Bits; import org.apache.lucene.util.SloppyMath; import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.common.unit.DistanceUnit; @@ -511,35 +510,40 @@ public static SortedNumericDoubleValues distanceValues(final GeoDistance distanc final GeoPoint... fromPoints) { final GeoPointValues singleValues = FieldData.unwrapSingleton(geoPointValues); if (singleValues != null && fromPoints.length == 1) { - final Bits docsWithField = FieldData.unwrapSingletonBits(geoPointValues); return FieldData.singleton(new NumericDoubleValues() { @Override - public double get(int docID) { - if (docsWithField != null && !docsWithField.get(docID)) { - return 0d; - } - final GeoPoint to = singleValues.get(docID); + public boolean advanceExact(int doc) throws IOException { + return singleValues.advanceExact(doc); + } + + @Override + public double doubleValue() throws IOException { final GeoPoint from = fromPoints[0]; + final GeoPoint to = singleValues.geoPointValue(); return distance.calculate(from.lat(), from.lon(), to.lat(), to.lon(), unit); } - }, docsWithField); + }); } else { return new SortingNumericDoubleValues() { @Override - public void setDocument(int doc) { - geoPointValues.setDocument(doc); - resize(geoPointValues.count() * fromPoints.length); - int v = 0; - for (GeoPoint from : fromPoints) { - for (int i = 0; i < geoPointValues.count(); ++i) { - final GeoPoint point = geoPointValues.valueAt(i); - values[v] = distance.calculate(from.lat(), from.lon(), point.lat(), point.lon(), unit); - v++; + public boolean advanceExact(int target) throws IOException { + if (geoPointValues.advanceExact(target)) { + resize(geoPointValues.docValueCount() * fromPoints.length); + int v = 0; + for (int i = 0; i < geoPointValues.docValueCount(); ++i) { + for (GeoPoint from : fromPoints) { + final GeoPoint point = geoPointValues.nextValue(); + values[v] = distance.calculate(from.lat(), from.lon(), point.lat(), point.lon(), unit); + v++; + } } + sort(); + return true; + } else { + return false; } - sort(); } }; } From 9e6e82e2f0e36af8716145f118c4d6ce8f2d4206 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Mon, 10 Apr 2017 10:58:52 +0200 Subject: [PATCH 39/87] Catch IOException at the highest possible level. --- .../search/fetch/FetchPhase.java | 44 +++++++++---------- .../search/fetch/FetchSubPhase.java | 5 ++- .../subphase/DocValueFieldsFetchSubPhase.java | 3 +- .../search/lookup/LeafDocLookup.java | 8 +++- 4 files changed, 34 insertions(+), 26 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java b/core/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java index 83af0b9abd4cc..edf7e44c396cb 100644 --- a/core/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java +++ b/core/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java @@ -132,41 +132,41 @@ public void execute(SearchContext context) { } } - SearchHit[] hits = new SearchHit[context.docIdsToLoadSize()]; - FetchSubPhase.HitContext hitContext = new FetchSubPhase.HitContext(); - for (int index = 0; index < context.docIdsToLoadSize(); index++) { - if(context.isCancelled()) { - throw new TaskCancelledException("cancelled"); - } - int docId = context.docIdsToLoad()[context.docIdsToLoadFrom() + index]; - int readerIndex = ReaderUtil.subIndex(docId, context.searcher().getIndexReader().leaves()); - LeafReaderContext subReaderContext = context.searcher().getIndexReader().leaves().get(readerIndex); - int subDocId = docId - subReaderContext.docBase; + try { + SearchHit[] hits = new SearchHit[context.docIdsToLoadSize()]; + FetchSubPhase.HitContext hitContext = new FetchSubPhase.HitContext(); + for (int index = 0; index < context.docIdsToLoadSize(); index++) { + if(context.isCancelled()) { + throw new TaskCancelledException("cancelled"); + } + int docId = context.docIdsToLoad()[context.docIdsToLoadFrom() + index]; + int readerIndex = ReaderUtil.subIndex(docId, context.searcher().getIndexReader().leaves()); + LeafReaderContext subReaderContext = context.searcher().getIndexReader().leaves().get(readerIndex); + int subDocId = docId - subReaderContext.docBase; - final SearchHit searchHit; - try { + final SearchHit searchHit; int rootDocId = findRootDocumentIfNested(context, subReaderContext, subDocId); if (rootDocId != -1) { searchHit = createNestedSearchHit(context, docId, subDocId, rootDocId, fieldNames, fieldNamePatterns, subReaderContext); } else { searchHit = createSearchHit(context, fieldsVisitor, docId, subDocId, subReaderContext); } - } catch (IOException e) { - throw ExceptionsHelper.convertToElastic(e); + + hits[index] = searchHit; + hitContext.reset(searchHit, subReaderContext, subDocId, context.searcher()); + for (FetchSubPhase fetchSubPhase : fetchSubPhases) { + fetchSubPhase.hitExecute(context, hitContext); + } } - hits[index] = searchHit; - hitContext.reset(searchHit, subReaderContext, subDocId, context.searcher()); for (FetchSubPhase fetchSubPhase : fetchSubPhases) { - fetchSubPhase.hitExecute(context, hitContext); + fetchSubPhase.hitsExecute(context, hits); } - } - for (FetchSubPhase fetchSubPhase : fetchSubPhases) { - fetchSubPhase.hitsExecute(context, hits); + context.fetchResult().hits(new SearchHits(hits, context.queryResult().topDocs().totalHits, context.queryResult().topDocs().getMaxScore())); + } catch (IOException e) { + throw ExceptionsHelper.convertToElastic(e); } - - context.fetchResult().hits(new SearchHits(hits, context.queryResult().topDocs().totalHits, context.queryResult().topDocs().getMaxScore())); } private int findRootDocumentIfNested(SearchContext context, LeafReaderContext subReaderContext, int subDocId) throws IOException { diff --git a/core/src/main/java/org/elasticsearch/search/fetch/FetchSubPhase.java b/core/src/main/java/org/elasticsearch/search/fetch/FetchSubPhase.java index 6f34eba2129b8..84154926bf665 100644 --- a/core/src/main/java/org/elasticsearch/search/fetch/FetchSubPhase.java +++ b/core/src/main/java/org/elasticsearch/search/fetch/FetchSubPhase.java @@ -25,6 +25,7 @@ import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.internal.SearchContext; +import java.io.IOException; import java.util.HashMap; import java.util.Map; @@ -79,8 +80,8 @@ public Map cache() { /** * Executes the hit level phase, with a reader and doc id (note, its a low level reader, and the matching doc). */ - default void hitExecute(SearchContext context, HitContext hitContext) {} + default void hitExecute(SearchContext context, HitContext hitContext) throws IOException {} - default void hitsExecute(SearchContext context, SearchHit[] hits) {} + default void hitsExecute(SearchContext context, SearchHit[] hits) throws IOException {} } diff --git a/core/src/main/java/org/elasticsearch/search/fetch/subphase/DocValueFieldsFetchSubPhase.java b/core/src/main/java/org/elasticsearch/search/fetch/subphase/DocValueFieldsFetchSubPhase.java index 031f7a6240030..42cee23d390cf 100644 --- a/core/src/main/java/org/elasticsearch/search/fetch/subphase/DocValueFieldsFetchSubPhase.java +++ b/core/src/main/java/org/elasticsearch/search/fetch/subphase/DocValueFieldsFetchSubPhase.java @@ -25,6 +25,7 @@ import org.elasticsearch.search.fetch.FetchSubPhase; import org.elasticsearch.search.internal.SearchContext; +import java.io.IOException; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -37,7 +38,7 @@ public final class DocValueFieldsFetchSubPhase implements FetchSubPhase { @Override - public void hitExecute(SearchContext context, HitContext hitContext) { + public void hitExecute(SearchContext context, HitContext hitContext) throws IOException { if (context.collapse() != null) { // retrieve the `doc_value` associated with the collapse field String name = context.collapse().getFieldType().name(); diff --git a/core/src/main/java/org/elasticsearch/search/lookup/LeafDocLookup.java b/core/src/main/java/org/elasticsearch/search/lookup/LeafDocLookup.java index 6ce060c493e1e..f02cf14f4ba26 100644 --- a/core/src/main/java/org/elasticsearch/search/lookup/LeafDocLookup.java +++ b/core/src/main/java/org/elasticsearch/search/lookup/LeafDocLookup.java @@ -19,12 +19,14 @@ package org.elasticsearch.search.lookup; import org.apache.lucene.index.LeafReaderContext; +import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.common.Nullable; import org.elasticsearch.index.fielddata.IndexFieldDataService; import org.elasticsearch.index.fielddata.ScriptDocValues; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperService; +import java.io.IOException; import java.security.AccessController; import java.security.PrivilegedAction; import java.util.Arrays; @@ -86,7 +88,11 @@ public ScriptDocValues run() { }); localCacheFieldData.put(fieldName, scriptValues); } - scriptValues.setNextDocId(docId); + try { + scriptValues.setNextDocId(docId); + } catch (IOException e) { + throw ExceptionsHelper.convertToElastic(e); + } return scriptValues; } From d6719cb10e7eae6d4b292fddee496b88e209db54 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Mon, 10 Apr 2017 11:17:33 +0200 Subject: [PATCH 40/87] Cut over to the new fielddata APIs in ScriptSortBuilder. --- .../search/sort/ScriptSortBuilder.java | 23 +++++++++++++------ 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/search/sort/ScriptSortBuilder.java b/core/src/main/java/org/elasticsearch/search/sort/ScriptSortBuilder.java index ec4e19d1eea56..2901e05f05191 100644 --- a/core/src/main/java/org/elasticsearch/search/sort/ScriptSortBuilder.java +++ b/core/src/main/java/org/elasticsearch/search/sort/ScriptSortBuilder.java @@ -32,6 +32,7 @@ import org.elasticsearch.common.xcontent.ConstructingObjectParser; import org.elasticsearch.common.xcontent.ObjectParser.ValueType; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.index.fielddata.AbstractBinaryDocValues; import org.elasticsearch.index.fielddata.FieldData; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested; @@ -261,16 +262,20 @@ public SortFieldAndFormat build(QueryShardContext context) throws IOException { @Override protected SortedBinaryDocValues getValues(LeafReaderContext context) throws IOException { leafScript = searchScript.getLeafSearchScript(context); - final BinaryDocValues values = new BinaryDocValues() { + final BinaryDocValues values = new AbstractBinaryDocValues() { final BytesRefBuilder spare = new BytesRefBuilder(); @Override - public BytesRef get(int docID) { - leafScript.setDocument(docID); + public boolean advanceExact(int doc) throws IOException { + leafScript.setDocument(doc); + return true; + } + @Override + public BytesRef binaryValue() { spare.copyChars(leafScript.run().toString()); return spare.get(); } }; - return FieldData.singleton(values, null); + return FieldData.singleton(values); } @Override protected void setScorer(Scorer scorer) { @@ -286,12 +291,16 @@ protected SortedNumericDoubleValues getValues(LeafReaderContext context) throws leafScript = searchScript.getLeafSearchScript(context); final NumericDoubleValues values = new NumericDoubleValues() { @Override - public double get(int docID) { - leafScript.setDocument(docID); + public boolean advanceExact(int doc) throws IOException { + leafScript.setDocument(doc); + return false; + } + @Override + public double doubleValue() { return leafScript.runAsDouble(); } }; - return FieldData.singleton(values, null); + return FieldData.singleton(values); } @Override protected void setScorer(Scorer scorer) { From 4ff2335b8e3f51da62705a7c0b68d18627122d66 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Mon, 10 Apr 2017 11:18:56 +0200 Subject: [PATCH 41/87] Add missing io exceptions. --- .../index/fielddata/ScriptDocValuesDatesTests.java | 4 +++- .../index/fielddata/ScriptDocValuesGeoPointsTests.java | 5 +++-- .../index/fielddata/ScriptDocValuesLongsTests.java | 6 ++++-- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/core/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesDatesTests.java b/core/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesDatesTests.java index ed1b189e5f44b..626327d454910 100644 --- a/core/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesDatesTests.java +++ b/core/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesDatesTests.java @@ -25,8 +25,10 @@ import org.joda.time.DateTimeZone; import org.joda.time.ReadableDateTime; +import java.io.IOException; + public class ScriptDocValuesDatesTests extends ESTestCase { - public void test() { + public void test() throws IOException { long[][] values = new long[between(3, 10)][]; ReadableDateTime[][] expectedDates = new ReadableDateTime[values.length][]; for (int d = 0; d < values.length; d++) { diff --git a/core/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesGeoPointsTests.java b/core/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesGeoPointsTests.java index dfa19d102b981..dab348f89d619 100644 --- a/core/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesGeoPointsTests.java +++ b/core/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesGeoPointsTests.java @@ -23,6 +23,7 @@ import org.elasticsearch.common.geo.GeoUtils; import org.elasticsearch.test.ESTestCase; +import java.io.IOException; import java.util.Arrays; public class ScriptDocValuesGeoPointsTests extends ESTestCase { @@ -63,7 +64,7 @@ private static double randomLon() { return randomDouble() * 360 - 180; } - public void testGeoGetLatLon() { + public void testGeoGetLatLon() throws IOException { final double lat1 = randomLat(); final double lat2 = randomLat(); final double lon1 = randomLon(); @@ -82,7 +83,7 @@ public void testGeoGetLatLon() { assertTrue(Arrays.equals(new double[] {lon1, lon2}, script.getLons())); } - public void testGeoDistance() { + public void testGeoDistance() throws IOException { final double lat = randomLat(); final double lon = randomLon(); final MultiGeoPointValues values = wrap(new GeoPoint(lat, lon)); diff --git a/core/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesLongsTests.java b/core/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesLongsTests.java index d1986be846347..3a560a4f629f5 100644 --- a/core/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesLongsTests.java +++ b/core/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesLongsTests.java @@ -25,8 +25,10 @@ import org.joda.time.DateTimeZone; import org.joda.time.ReadableDateTime; +import java.io.IOException; + public class ScriptDocValuesLongsTests extends ESTestCase { - public void testLongs() { + public void testLongs() throws IOException { long[][] values = new long[between(3, 10)][]; for (int d = 0; d < values.length; d++) { values[d] = new long[randomBoolean() ? randomBoolean() ? 0 : 1 : between(2, 100)]; @@ -53,7 +55,7 @@ public void testLongs() { } } - public void testDates() { + public void testDates() throws IOException { long[][] values = new long[between(3, 10)][]; ReadableDateTime[][] dates = new ReadableDateTime[values.length][]; for (int d = 0; d < values.length; d++) { From 609ae4ed003e453316371627e48cee08401ebd30 Mon Sep 17 00:00:00 2001 From: Colin Goodheart-Smithe Date: Fri, 7 Apr 2017 14:20:45 +0000 Subject: [PATCH 42/87] Addressed comments for ScriptDocValues --- .../index/fielddata/ScriptDocValues.java | 55 +++++++++---------- 1 file changed, 26 insertions(+), 29 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java b/core/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java index ce038298ce0c7..a50fb91776004 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java @@ -35,6 +35,7 @@ import java.io.IOException; import java.util.AbstractList; +import java.util.Arrays; import java.util.Comparator; import java.util.List; import java.util.function.UnaryOperator; @@ -86,7 +87,7 @@ public final void sort(Comparator c) { public static final class Strings extends ScriptDocValues { private final SortedBinaryDocValues in; - private BytesRef[] values; + private BytesRef[] values = new BytesRef[0]; private int count; public Strings(SortedBinaryDocValues in) { @@ -111,11 +112,7 @@ public void setNextDocId(int docId) throws IOException { */ protected final void resize(int newSize) { count = newSize; - if (values == null) { - values = new BytesRef[newSize]; - } else { - values = ArrayUtil.grow(values, count); - } + values = ArrayUtil.grow(values, count); } public SortedBinaryDocValues getInternalValues() { @@ -155,7 +152,7 @@ public static final class Longs extends ScriptDocValues { protected static final DeprecationLogger deprecationLogger = new DeprecationLogger(ESLoggerFactory.getLogger(Longs.class)); private final SortedNumericDocValues in; - private Long[] values; + private long[] values = new long[0]; private int count; private Dates dates; @@ -185,11 +182,7 @@ public void setNextDocId(int docId) throws IOException { */ protected final void resize(int newSize) { count = newSize; - if (values == null) { - values = new Long[newSize]; - } else { - values = ArrayUtil.grow(values, count); - } + values = ArrayUtil.grow(values, count); } public SortedNumericDocValues getInternalValues() { @@ -342,7 +335,7 @@ void refreshArray() throws IOException { public static final class Doubles extends ScriptDocValues { private final SortedNumericDoubleValues in; - private Double[] values; + private double[] values = new double[0]; private int count; public Doubles(SortedNumericDoubleValues in) { @@ -367,11 +360,7 @@ public void setNextDocId(int docId) throws IOException { */ protected final void resize(int newSize) { count = newSize; - if (values == null) { - values = new Double[newSize]; - } else { - values = ArrayUtil.grow(values, count); - } + values = ArrayUtil.grow(values, count); } public SortedNumericDoubleValues getInternalValues() { @@ -399,7 +388,7 @@ public int size() { public static final class GeoPoints extends ScriptDocValues { private final MultiGeoPointValues in; - private GeoPoint[] values; + private GeoPoint[] values = new GeoPoint[0]; private int count; public GeoPoints(MultiGeoPointValues in) { @@ -411,7 +400,8 @@ public void setNextDocId(int docId) throws IOException { if (in.advanceExact(docId)) { resize(in.docValueCount()); for (int i = 0; i < count; i++) { - values[i] = in.nextValue(); + GeoPoint point = in.nextValue(); + values[i].reset(point.lat(), point.lon()); } } else { resize(0); @@ -424,10 +414,12 @@ public void setNextDocId(int docId) throws IOException { */ protected final void resize(int newSize) { count = newSize; - if (values == null) { - values = new GeoPoint[newSize]; - } else { + if (newSize > values.length) { + int oldLength = values.length; values = ArrayUtil.grow(values, count); + for (int i = oldLength; i < values.length; ++i) { + values[i] = new GeoPoint(); + } } } @@ -516,7 +508,7 @@ public double geohashDistanceWithDefault(String geohash, double defaultValue) { public static final class Booleans extends ScriptDocValues { private final SortedNumericDocValues in; - private Boolean[] values; + private boolean[] values = new boolean[0]; private int count; public Booleans(SortedNumericDocValues in) { @@ -541,11 +533,7 @@ public void setNextDocId(int docId) throws IOException { */ protected final void resize(int newSize) { count = newSize; - if (values == null) { - values = new Boolean[newSize]; - } else { - values = ArrayUtil.grow(values, count); - } + values = grow(values, count); } public boolean getValue() { @@ -562,6 +550,15 @@ public int size() { return count; } + private static boolean[] grow(boolean[] array, int minSize) { + assert minSize >= 0 : "size must be positive (got " + minSize + + "): likely integer overflow?"; + if (array.length < minSize) { + return Arrays.copyOf(array, ArrayUtil.oversize(minSize, 1)); + } else + return array; + } + } public static final class BytesRefs extends ScriptDocValues { From ee4dd0060210fb39b5838e0d19fa03b9edadea96 Mon Sep 17 00:00:00 2001 From: Colin Goodheart-Smithe Date: Mon, 10 Apr 2017 12:02:56 +0100 Subject: [PATCH 43/87] Fixes compile errors in InnerHitsContext --- .../search/fetch/subphase/InnerHitsContext.java | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/search/fetch/subphase/InnerHitsContext.java b/core/src/main/java/org/elasticsearch/search/fetch/subphase/InnerHitsContext.java index 5c76328610dce..f6dcee9964de2 100644 --- a/core/src/main/java/org/elasticsearch/search/fetch/subphase/InnerHitsContext.java +++ b/core/src/main/java/org/elasticsearch/search/fetch/subphase/InnerHitsContext.java @@ -48,9 +48,9 @@ import org.elasticsearch.index.mapper.ParentFieldMapper; import org.elasticsearch.index.mapper.Uid; import org.elasticsearch.index.mapper.UidFieldMapper; +import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHitField; import org.elasticsearch.search.fetch.FetchSubPhase; -import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.internal.SearchContext; import org.elasticsearch.search.internal.SubSearchContext; @@ -180,7 +180,8 @@ public boolean equals(Object obj) { return parentFilter.equals(other.parentFilter) && childFilter.equals(other.childFilter) && docId == other.docId - && leafReader.getCoreCacheKey() == other.leafReader.getCoreCacheKey(); + && leafReader.getCoreCacheHelper().getKey() == other.leafReader + .getCoreCacheHelper().getKey(); } @Override @@ -189,7 +190,7 @@ public int hashCode() { hash = 31 * hash + parentFilter.hashCode(); hash = 31 * hash + childFilter.hashCode(); hash = 31 * hash + docId; - hash = 31 * hash + leafReader.getCoreCacheKey().hashCode(); + hash = 31 * hash + leafReader.getCoreCacheHelper().getKey().hashCode(); return hash; } @@ -199,13 +200,15 @@ public String toString(String field) { } @Override - public Weight createWeight(IndexSearcher searcher, boolean needsScores) throws IOException { - final Weight childWeight = childFilter.createWeight(searcher, false); - return new ConstantScoreWeight(this) { + public Weight createWeight(IndexSearcher searcher, boolean needsScores, float boost) + throws IOException { + final Weight childWeight = childFilter.createWeight(searcher, false, boost); + return new ConstantScoreWeight(this, boost) { @Override public Scorer scorer(LeafReaderContext context) throws IOException { // Nested docs only reside in a single segment, so no need to evaluate all segments - if (!context.reader().getCoreCacheKey().equals(leafReader.getCoreCacheKey())) { + if (!context.reader().getCoreCacheHelper().getKey() + .equals(leafReader.getCoreCacheHelper().getKey())) { return null; } From 237f39dd3051b1aa30dca1700362f40ede0ad2ad Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Tue, 11 Apr 2017 11:14:29 +0200 Subject: [PATCH 44/87] Remove forbidden signatures of removed methods. --- .../src/main/resources/forbidden/es-all-signatures.txt | 6 ------ .../main/resources/forbidden/es-core-signatures.txt | 10 ---------- 2 files changed, 16 deletions(-) diff --git a/buildSrc/src/main/resources/forbidden/es-all-signatures.txt b/buildSrc/src/main/resources/forbidden/es-all-signatures.txt index 64ae6784f15f2..f1d271d602ce1 100644 --- a/buildSrc/src/main/resources/forbidden/es-all-signatures.txt +++ b/buildSrc/src/main/resources/forbidden/es-all-signatures.txt @@ -26,12 +26,6 @@ java.util.concurrent.ThreadLocalRandom java.security.MessageDigest#clone() @ use org.elasticsearch.common.hash.MessageDigests -@defaultMessage this should not have been added to lucene in the first place -org.apache.lucene.index.IndexReader#getCombinedCoreAndDeletesKey() - -@defaultMessage Soon to be removed -org.apache.lucene.document.FieldType#numericType() - @defaultMessage Don't use MethodHandles in slow ways, don't be lenient in tests. java.lang.invoke.MethodHandle#invoke(java.lang.Object[]) java.lang.invoke.MethodHandle#invokeWithArguments(java.lang.Object[]) diff --git a/buildSrc/src/main/resources/forbidden/es-core-signatures.txt b/buildSrc/src/main/resources/forbidden/es-core-signatures.txt index 059be403a672f..6507f05be5cd3 100644 --- a/buildSrc/src/main/resources/forbidden/es-core-signatures.txt +++ b/buildSrc/src/main/resources/forbidden/es-core-signatures.txt @@ -36,16 +36,6 @@ org.apache.lucene.index.IndexReader#decRef() org.apache.lucene.index.IndexReader#incRef() org.apache.lucene.index.IndexReader#tryIncRef() -@defaultMessage Close listeners can only installed via ElasticsearchDirectoryReader#addReaderCloseListener -org.apache.lucene.index.IndexReader#addReaderClosedListener(org.apache.lucene.index.IndexReader$ReaderClosedListener) -org.apache.lucene.index.IndexReader#removeReaderClosedListener(org.apache.lucene.index.IndexReader$ReaderClosedListener) - -@defaultMessage Pass the precision step from the mappings explicitly instead -org.apache.lucene.search.LegacyNumericRangeQuery#newDoubleRange(java.lang.String,java.lang.Double,java.lang.Double,boolean,boolean) -org.apache.lucene.search.LegacyNumericRangeQuery#newFloatRange(java.lang.String,java.lang.Float,java.lang.Float,boolean,boolean) -org.apache.lucene.search.LegacyNumericRangeQuery#newIntRange(java.lang.String,java.lang.Integer,java.lang.Integer,boolean,boolean) -org.apache.lucene.search.LegacyNumericRangeQuery#newLongRange(java.lang.String,java.lang.Long,java.lang.Long,boolean,boolean) - @defaultMessage Only use wait / notify when really needed try to use concurrency primitives, latches or callbacks instead. java.lang.Object#wait() java.lang.Object#wait(long) From 85156afd37d4fe8aa341298708746e9089969475 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Tue, 11 Apr 2017 11:14:53 +0200 Subject: [PATCH 45/87] Fix bad javadocs links. --- .../elasticsearch/index/fielddata/FieldData.java | 9 +++------ .../index/fielddata/MultiGeoPointValues.java | 2 +- .../fielddata/SortedNumericDoubleValues.java | 5 ++--- .../index/fielddata/SortingNumericDocValues.java | 2 +- .../fielddata/SortingNumericDoubleValues.java | 4 ++-- .../fielddata/ordinals/GlobalOrdinalMapping.java | 2 +- .../fielddata/ordinals/OrdinalsBuilder.java | 8 -------- .../plain/SortedNumericDVIndexFieldData.java | 16 ++++------------ .../index/query/CommonTermsQueryBuilder.java | 10 +--------- .../index/shard/IndexSearcherWrapper.java | 5 +++-- 10 files changed, 18 insertions(+), 45 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/FieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/FieldData.java index 86e375ad95f50..31be1b55a28f1 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/FieldData.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/FieldData.java @@ -282,9 +282,8 @@ public static SortedNumericDoubleValues singleton(NumericDoubleValues values) { /** * Returns a single-valued view of the {@link SortedNumericDoubleValues}, - * if it was previously wrapped with {@link DocValues#singleton(NumericDocValues, Bits)}, + * if it was previously wrapped with {@link DocValues#singleton(NumericDocValues)}, * or null. - * @see DocValues#unwrapSingletonBits(SortedNumericDocValues) */ public static NumericDoubleValues unwrapSingleton(SortedNumericDoubleValues values) { if (values instanceof SingletonSortedNumericDoubleValues) { @@ -302,9 +301,8 @@ public static MultiGeoPointValues singleton(GeoPointValues values) { /** * Returns a single-valued view of the {@link MultiGeoPointValues}, - * if it was previously wrapped with {@link #singleton(GeoPointValues, Bits)}, + * if it was previously wrapped with {@link #singleton(GeoPointValues)}, * or null. - * @see #unwrapSingletonBits(MultiGeoPointValues) */ public static GeoPointValues unwrapSingleton(MultiGeoPointValues values) { if (values instanceof SingletonMultiGeoPointValues) { @@ -322,9 +320,8 @@ public static SortedBinaryDocValues singleton(BinaryDocValues values) { /** * Returns a single-valued view of the {@link SortedBinaryDocValues}, - * if it was previously wrapped with {@link #singleton(BinaryDocValues, Bits)}, + * if it was previously wrapped with {@link #singleton(BinaryDocValues)}, * or null. - * @see #unwrapSingletonBits(SortedBinaryDocValues) */ public static BinaryDocValues unwrapSingleton(SortedBinaryDocValues values) { if (values instanceof SingletonSortedBinaryDocValues) { diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/MultiGeoPointValues.java b/core/src/main/java/org/elasticsearch/index/fielddata/MultiGeoPointValues.java index 0e2bfcecdd157..82bcd8acb6702 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/MultiGeoPointValues.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/MultiGeoPointValues.java @@ -62,7 +62,7 @@ protected MultiGeoPointValues() { * * Note: the returned {@link GeoPoint} might be shared across invocations. * - * @return the next value for the current docID set to {@link #setDocument(int)}. + * @return the next value for the current docID set to {@link #advanceExact(int)}. */ public abstract GeoPoint nextValue(); diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/SortedNumericDoubleValues.java b/core/src/main/java/org/elasticsearch/index/fielddata/SortedNumericDoubleValues.java index e036e9dd68896..d0d9fc4b4c79e 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/SortedNumericDoubleValues.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/SortedNumericDoubleValues.java @@ -35,9 +35,8 @@ protected SortedNumericDoubleValues() {} /** Advance the iterator to exactly {@code target} and return whether * {@code target} has a value. * {@code target} must be greater than or equal to the current - * {@link #docID() doc ID} and must be a valid doc ID, ie. ≥ 0 and - * < {@code maxDoc}. - * After this method returns, {@link #docID()} retuns {@code target}. */ + * doc ID and must be a valid doc ID, ie. ≥ 0 and + * < {@code maxDoc}.*/ public abstract boolean advanceExact(int target) throws IOException; /** diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/SortingNumericDocValues.java b/core/src/main/java/org/elasticsearch/index/fielddata/SortingNumericDocValues.java index 082892dce2ab7..0049faaf2b5ed 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/SortingNumericDocValues.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/SortingNumericDocValues.java @@ -54,7 +54,7 @@ protected int compare(int i, int j) { } /** - * Set the {@link #count()} and ensure that the {@link #values} array can + * Set the {@link #docValueCount()} and ensure that the {@link #values} array can * store at least that many entries. */ protected final void resize(int newSize) { diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/SortingNumericDoubleValues.java b/core/src/main/java/org/elasticsearch/index/fielddata/SortingNumericDoubleValues.java index c3aac671555e8..7a39f64745613 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/SortingNumericDoubleValues.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/SortingNumericDoubleValues.java @@ -53,7 +53,7 @@ protected int compare(int i, int j) { } /** - * Set the {@link #count()} and ensure that the {@link #values} array can + * Set the {@link #docValueCount()} and ensure that the {@link #values} array can * store at least that many entries. */ protected final void resize(int newSize) { @@ -64,7 +64,7 @@ protected final void resize(int newSize) { /** * Sort values that are stored between offsets 0 and - * {@link #count} of {@link #values}. + * {@link #docValueCount} of {@link #values}. */ protected final void sort() { sorter.sort(0, count); diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/GlobalOrdinalMapping.java b/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/GlobalOrdinalMapping.java index 65f70e85d50e4..3eda112674dfc 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/GlobalOrdinalMapping.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/GlobalOrdinalMapping.java @@ -27,7 +27,7 @@ import java.io.IOException; /** - * A {@link RandomAccessOrds} implementation that returns ordinals that are global. + * A {@link SortedSetDocValues} implementation that returns ordinals that are global. */ public class GlobalOrdinalMapping extends SortedSetDocValues { diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/OrdinalsBuilder.java b/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/OrdinalsBuilder.java index 516530ae73e10..b2e1b9e3104fb 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/OrdinalsBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/OrdinalsBuilder.java @@ -425,14 +425,6 @@ public int maxDoc() { * exhaust the returned {@link BytesRefIterator} which returns all values * where the first returned value is associated with the ordinal 1 * etc. - *

- * If the {@link TermsEnum} contains prefix coded numerical values the terms - * enum should be wrapped with either {@link #wrapNumeric32Bit(TermsEnum)} - * or {@link #wrapNumeric64Bit(TermsEnum)} depending on its precision. If - * the {@link TermsEnum} is not wrapped the returned - * {@link BytesRefIterator} will contain partial precision terms rather than - * only full-precision terms. - *

*/ public BytesRefIterator buildFromTerms(final TermsEnum termsEnum) throws IOException { return new BytesRefIterator() { diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/SortedNumericDVIndexFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/SortedNumericDVIndexFieldData.java index 9be850651fe3c..1b28d3a8c7bc0 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/SortedNumericDVIndexFieldData.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/plain/SortedNumericDVIndexFieldData.java @@ -148,9 +148,7 @@ public AtomicNumericFieldData load(LeafReaderContext context) { * Although the API is multi-valued, most codecs in Lucene specialize * for the case where documents have at most one value. In this case * {@link DocValues#unwrapSingleton(SortedNumericDocValues)} will return - * the underlying single-valued NumericDocValues representation, and - * {@link DocValues#unwrapSingletonBits(SortedNumericDocValues)} will return - * a Bits matching documents that have a real value (as opposed to missing). + * the underlying single-valued NumericDocValues representation. */ static final class SortedNumericLongFieldData extends AtomicLongFieldData { final LeafReader reader; @@ -188,9 +186,7 @@ public Collection getChildResources() { * Although the API is multi-valued, most codecs in Lucene specialize * for the case where documents have at most one value. In this case * {@link FieldData#unwrapSingleton(SortedNumericDoubleValues)} will return - * the underlying single-valued NumericDoubleValues representation, and - * {@link FieldData#unwrapSingletonBits(SortedNumericDoubleValues)} will return - * a Bits matching documents that have a real value (as opposed to missing). + * the underlying single-valued NumericDoubleValues representation. */ static final class SortedNumericHalfFloatFieldData extends AtomicDoubleFieldData { final LeafReader reader; @@ -282,9 +278,7 @@ public int docValueCount() { * Although the API is multi-valued, most codecs in Lucene specialize * for the case where documents have at most one value. In this case * {@link FieldData#unwrapSingleton(SortedNumericDoubleValues)} will return - * the underlying single-valued NumericDoubleValues representation, and - * {@link FieldData#unwrapSingletonBits(SortedNumericDoubleValues)} will return - * a Bits matching documents that have a real value (as opposed to missing). + * the underlying single-valued NumericDoubleValues representation. */ static final class SortedNumericFloatFieldData extends AtomicDoubleFieldData { final LeafReader reader; @@ -376,9 +370,7 @@ public int docValueCount() { * Although the API is multi-valued, most codecs in Lucene specialize * for the case where documents have at most one value. In this case * {@link FieldData#unwrapSingleton(SortedNumericDoubleValues)} will return - * the underlying single-valued NumericDoubleValues representation, and - * {@link FieldData#unwrapSingletonBits(SortedNumericDoubleValues)} will return - * a Bits matching documents that have a real value (as opposed to missing). + * the underlying single-valued NumericDoubleValues representation. */ static final class SortedNumericDoubleFieldData extends AtomicDoubleFieldData { final LeafReader reader; diff --git a/core/src/main/java/org/elasticsearch/index/query/CommonTermsQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/CommonTermsQueryBuilder.java index cd2a9c1193940..e5fff943cd39a 100644 --- a/core/src/main/java/org/elasticsearch/index/query/CommonTermsQueryBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/query/CommonTermsQueryBuilder.java @@ -25,9 +25,7 @@ import org.apache.lucene.index.Term; import org.apache.lucene.queries.ExtendedCommonTermsQuery; import org.apache.lucene.search.BooleanClause.Occur; -import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.Query; -import org.apache.lucene.search.similarities.Similarity; import org.apache.lucene.util.BytesRefBuilder; import org.elasticsearch.Version; import org.elasticsearch.common.ParseField; @@ -49,13 +47,7 @@ * added} terms where low-frequency terms are added to a required boolean clause * and high-frequency terms are added to an optional boolean clause. The * optional clause is only executed if the required "low-frequency' clause - * matches. Scores produced by this query will be slightly different to plain - * {@link BooleanQuery} scorer mainly due to differences in the - * {@link Similarity#coord(int,int) number of leave queries} in the required - * boolean clause. In the most cases high-frequency terms are unlikely to - * significantly contribute to the document score unless at least one of the - * low-frequency terms are matched such that this query can improve query - * execution times significantly if applicable. + * matches. */ public class CommonTermsQueryBuilder extends AbstractQueryBuilder { diff --git a/core/src/main/java/org/elasticsearch/index/shard/IndexSearcherWrapper.java b/core/src/main/java/org/elasticsearch/index/shard/IndexSearcherWrapper.java index 5f6065ca66978..a2e738128e3eb 100644 --- a/core/src/main/java/org/elasticsearch/index/shard/IndexSearcherWrapper.java +++ b/core/src/main/java/org/elasticsearch/index/shard/IndexSearcherWrapper.java @@ -40,8 +40,9 @@ public class IndexSearcherWrapper { * Wraps the given {@link DirectoryReader}. The wrapped reader can filter out document just like delete documents etc. but * must not change any term or document content. *

- * NOTE: The wrapper has a per-request lifecycle, must delegate {@link IndexReader#getCoreCacheKey()} and must be an instance - * of {@link FilterDirectoryReader} that eventually exposes the original reader via {@link FilterDirectoryReader#getDelegate()}. + * NOTE: The wrapper has a per-request lifecycle, must delegate {@link IndexReader#getReaderCacheHelper()}, + * {@link LeafReader#getCoreCacheHelper()} and must be an instance of {@link FilterDirectoryReader} that + * eventually exposes the original reader via {@link FilterDirectoryReader#getDelegate()}. * The returned reader is closed once it goes out of scope. *

* @param reader The provided directory reader to be wrapped to add custom functionality From d9765a600b557700f249fd1b55c2b8e8d45899d5 Mon Sep 17 00:00:00 2001 From: Colin Goodheart-Smithe Date: Tue, 11 Apr 2017 10:38:17 +0100 Subject: [PATCH 46/87] fixes checkstyle errors --- .../org/elasticsearch/common/geo/GeoHashUtils.java | 10 +++++----- .../org/elasticsearch/index/fielddata/FieldData.java | 2 +- .../index/fielddata/ScriptDocValues.java | 12 ++++++------ 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/common/geo/GeoHashUtils.java b/core/src/main/java/org/elasticsearch/common/geo/GeoHashUtils.java index edac846bc5e43..cb31940a49c0d 100644 --- a/core/src/main/java/org/elasticsearch/common/geo/GeoHashUtils.java +++ b/core/src/main/java/org/elasticsearch/common/geo/GeoHashUtils.java @@ -18,13 +18,13 @@ */ package org.elasticsearch.common.geo; -import java.util.ArrayList; -import java.util.Collection; - import org.apache.lucene.geo.Rectangle; import org.apache.lucene.spatial.util.MortonEncoder; import org.apache.lucene.util.BitUtil; +import java.util.ArrayList; +import java.util.Collection; + /** * Utilities for converting to/from the GeoHash standard * @@ -329,11 +329,11 @@ public static final double decodeLatitude(final long hash) { return unscaleLat(BitUtil.deinterleave(hash >>> 1)); } - private static final double unscaleLon(final long val) { + private static double unscaleLon(final long val) { return (val / LON_SCALE) - 180; } - private static final double unscaleLat(final long val) { + private static double unscaleLat(final long val) { return (val / LAT_SCALE) - 90; } diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/FieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/FieldData.java index 31be1b55a28f1..06d3c8b5b2de3 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/FieldData.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/FieldData.java @@ -500,7 +500,7 @@ private interface ToStringValues { * Advance this instance to the given document id * @return true if there is a value for this document */ - public abstract boolean advanceExact(int doc) throws IOException; + abstract boolean advanceExact(int doc) throws IOException; /** Fill the list of charsquences with the list of values for the current document. */ void get(List values) throws IOException; diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java b/core/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java index a50fb91776004..5776f84a83ff5 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java @@ -110,7 +110,7 @@ public void setNextDocId(int docId) throws IOException { * Set the {@link #size()} and ensure that the {@link #values} array can * store at least that many entries. */ - protected final void resize(int newSize) { + protected void resize(int newSize) { count = newSize; values = ArrayUtil.grow(values, count); } @@ -180,7 +180,7 @@ public void setNextDocId(int docId) throws IOException { * Set the {@link #size()} and ensure that the {@link #values} array can * store at least that many entries. */ - protected final void resize(int newSize) { + protected void resize(int newSize) { count = newSize; values = ArrayUtil.grow(values, count); } @@ -358,7 +358,7 @@ public void setNextDocId(int docId) throws IOException { * Set the {@link #size()} and ensure that the {@link #values} array can * store at least that many entries. */ - protected final void resize(int newSize) { + protected void resize(int newSize) { count = newSize; values = ArrayUtil.grow(values, count); } @@ -412,7 +412,7 @@ public void setNextDocId(int docId) throws IOException { * Set the {@link #size()} and ensure that the {@link #values} array can * store at least that many entries. */ - protected final void resize(int newSize) { + protected void resize(int newSize) { count = newSize; if (newSize > values.length) { int oldLength = values.length; @@ -531,7 +531,7 @@ public void setNextDocId(int docId) throws IOException { * Set the {@link #size()} and ensure that the {@link #values} array can * store at least that many entries. */ - protected final void resize(int newSize) { + protected void resize(int newSize) { count = newSize; values = grow(values, count); } @@ -587,7 +587,7 @@ public void setNextDocId(int docId) throws IOException { * Set the {@link #size()} and ensure that the {@link #values} array can * store at least that many entries. */ - protected final void resize(int newSize) { + protected void resize(int newSize) { count = newSize; if (values == null) { values = new BytesRef[newSize]; From 0f6eeae66bdc9c0c688cc1b8f96f9e5bf016385e Mon Sep 17 00:00:00 2001 From: Colin Goodheart-Smithe Date: Tue, 11 Apr 2017 10:42:54 +0100 Subject: [PATCH 47/87] Fixes another checkstyle error --- .../main/java/org/elasticsearch/index/fielddata/FieldData.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/FieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/FieldData.java index 06d3c8b5b2de3..268a8198d0aa3 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/FieldData.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/FieldData.java @@ -500,7 +500,7 @@ private interface ToStringValues { * Advance this instance to the given document id * @return true if there is a value for this document */ - abstract boolean advanceExact(int doc) throws IOException; + boolean advanceExact(int doc) throws IOException; /** Fill the list of charsquences with the list of values for the current document. */ void get(List values) throws IOException; From f6bdb3f5caa50181f8ab5cbe2a1abeb77655e784 Mon Sep 17 00:00:00 2001 From: Colin Goodheart-Smithe Date: Tue, 11 Apr 2017 11:01:35 +0100 Subject: [PATCH 48/87] Upgrades Version constants to Lucene 7 --- core/src/main/java/org/elasticsearch/Version.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/org/elasticsearch/Version.java b/core/src/main/java/org/elasticsearch/Version.java index 7ec5888674dee..db601bbfc1b99 100644 --- a/core/src/main/java/org/elasticsearch/Version.java +++ b/core/src/main/java/org/elasticsearch/Version.java @@ -80,7 +80,7 @@ public class Version implements Comparable { public static final Version V_5_4_0_UNRELEASED = new Version(V_5_4_0_ID_UNRELEASED, org.apache.lucene.util.Version.LUCENE_6_5_0); public static final int V_6_0_0_alpha1_ID_UNRELEASED = 6000001; public static final Version V_6_0_0_alpha1_UNRELEASED = - new Version(V_6_0_0_alpha1_ID_UNRELEASED, org.apache.lucene.util.Version.LUCENE_6_5_0); + new Version(V_6_0_0_alpha1_ID_UNRELEASED, org.apache.lucene.util.Version.LUCENE_7_0_0); public static final Version CURRENT = V_6_0_0_alpha1_UNRELEASED; // unreleased versions must be added to the above list with the suffix _UNRELEASED (with the exception of CURRENT) From e0b20ffc1abac79fb2be34f5bd0d9d0a243a2b97 Mon Sep 17 00:00:00 2001 From: Colin Goodheart-Smithe Date: Tue, 11 Apr 2017 11:15:57 +0100 Subject: [PATCH 49/87] Updates lucene format versions --- .../main/java/org/elasticsearch/common/lucene/Lucene.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/common/lucene/Lucene.java b/core/src/main/java/org/elasticsearch/common/lucene/Lucene.java index bf448b6153947..c213c384611f5 100644 --- a/core/src/main/java/org/elasticsearch/common/lucene/Lucene.java +++ b/core/src/main/java/org/elasticsearch/common/lucene/Lucene.java @@ -51,14 +51,14 @@ import org.apache.lucene.search.Scorer; import org.apache.lucene.search.SimpleCollector; import org.apache.lucene.search.SortField; +import org.apache.lucene.search.SortedNumericSortField; +import org.apache.lucene.search.SortedSetSortField; import org.apache.lucene.search.TimeLimitingCollector; import org.apache.lucene.search.TopDocs; import org.apache.lucene.search.TopFieldDocs; import org.apache.lucene.search.TwoPhaseIterator; import org.apache.lucene.search.Weight; import org.apache.lucene.search.grouping.CollapseTopFieldDocs; -import org.apache.lucene.search.SortedNumericSortField; -import org.apache.lucene.search.SortedSetSortField; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexInput; @@ -89,9 +89,9 @@ import java.util.Objects; public class Lucene { - public static final String LATEST_DOC_VALUES_FORMAT = "Lucene54"; + public static final String LATEST_DOC_VALUES_FORMAT = "Lucene70"; public static final String LATEST_POSTINGS_FORMAT = "Lucene50"; - public static final String LATEST_CODEC = "Lucene62"; + public static final String LATEST_CODEC = "Lucene70"; static { Deprecated annotation = PostingsFormat.forName(LATEST_POSTINGS_FORMAT).getClass().getAnnotation(Deprecated.class); From 4f65ef0da9dba04e41714cc24adf857c0e2446b1 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Tue, 11 Apr 2017 14:20:54 +0200 Subject: [PATCH 50/87] Fix more failures. --- .../search/grouping/CollapsingDocValuesSource.java | 5 +++++ .../index/fielddata/AbstractSortedDocValues.java | 5 ----- .../fieldcomparator/BytesRefFieldComparatorSource.java | 5 +++++ .../index/fielddata/ordinals/MultiOrdinals.java | 7 +++++++ .../index/fielddata/ordinals/SinglePackedOrdinals.java | 7 +++++++ .../index/fielddata/plain/IndexIndexFieldData.java | 8 ++++++++ .../fielddata/plain/ParentChildIndexFieldData.java | 5 +++++ .../java/org/elasticsearch/search/MultiValueMode.java | 10 ++++++++++ .../org/elasticsearch/search/MultiValueModeTests.java | 7 +++++++ 9 files changed, 54 insertions(+), 5 deletions(-) diff --git a/core/src/main/java/org/apache/lucene/search/grouping/CollapsingDocValuesSource.java b/core/src/main/java/org/apache/lucene/search/grouping/CollapsingDocValuesSource.java index ad3ee51bc3494..09d79ee28f173 100644 --- a/core/src/main/java/org/apache/lucene/search/grouping/CollapsingDocValuesSource.java +++ b/core/src/main/java/org/apache/lucene/search/grouping/CollapsingDocValuesSource.java @@ -194,6 +194,11 @@ public boolean advanceExact(int target) throws IOException { } } + @Override + public int docID() { + return sorted.docID(); + } + @Override public int ordValue() { return ord; diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/AbstractSortedDocValues.java b/core/src/main/java/org/elasticsearch/index/fielddata/AbstractSortedDocValues.java index 1cf2bcf999161..b1ace7e932b88 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/AbstractSortedDocValues.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/AbstractSortedDocValues.java @@ -32,11 +32,6 @@ */ public abstract class AbstractSortedDocValues extends SortedDocValues { - @Override - public int docID() { - throw new UnsupportedOperationException(); - } - @Override public int nextDoc() throws IOException { throw new UnsupportedOperationException(); diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/BytesRefFieldComparatorSource.java b/core/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/BytesRefFieldComparatorSource.java index d9a486bf7de56..4be5282e79286 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/BytesRefFieldComparatorSource.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/BytesRefFieldComparatorSource.java @@ -185,6 +185,11 @@ public boolean advanceExact(int target) throws IOException { return in.advanceExact(target); } + @Override + public int docID() { + return in.docID(); + } + @Override public int getValueCount() { if (exists) { diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/MultiOrdinals.java b/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/MultiOrdinals.java index cb277faeebb41..642d6e0a78760 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/MultiOrdinals.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/MultiOrdinals.java @@ -116,6 +116,7 @@ private static class SingleDocs extends AbstractSortedDocValues { private final PackedLongValues ords; private final ValuesHolder values; + private int currentDoc; private long currentStartOffset; private long currentEndOffset; @@ -133,11 +134,17 @@ public int ordValue() { @Override public boolean advanceExact(int docId) throws IOException { + currentDoc = docId; currentStartOffset = docId != 0 ? endOffsets.get(docId - 1) : 0; currentEndOffset = endOffsets.get(docId); return currentStartOffset != currentEndOffset; } + @Override + public int docID() { + return currentDoc; + } + @Override public BytesRef lookupOrd(int ord) { return values.lookupOrd(ord); diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/SinglePackedOrdinals.java b/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/SinglePackedOrdinals.java index 492bff0ea9ca4..daedae519b0da 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/SinglePackedOrdinals.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/SinglePackedOrdinals.java @@ -68,6 +68,7 @@ private static class Docs extends AbstractSortedDocValues { private final PackedInts.Reader reader; private final ValuesHolder values; + private int currentDoc; private int currentOrd; Docs(SinglePackedOrdinals parent, ValuesHolder values) { @@ -93,8 +94,14 @@ public int ordValue() { @Override public boolean advanceExact(int docID) throws IOException { + currentDoc = docID; currentOrd = (int) (reader.get(docID) - 1); return currentOrd != -1; } + + @Override + public int docID() { + return currentDoc; + } } } diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/IndexIndexFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/IndexIndexFieldData.java index d83cb0cdadbc3..1b9f07bf8a5ee 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/IndexIndexFieldData.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/plain/IndexIndexFieldData.java @@ -81,6 +81,8 @@ public SortedSetDocValues getOrdinalsValues() { final BytesRef term = new BytesRef(index); final SortedDocValues sortedValues = new AbstractSortedDocValues() { + private int docID; + @Override public BytesRef lookupOrd(int ord) { return term; @@ -98,8 +100,14 @@ public int ordValue() { @Override public boolean advanceExact(int target) throws IOException { + docID = target; return true; } + + @Override + public int docID() { + return docID; + } }; return (SortedSetDocValues) DocValues.singleton(sortedValues); } diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/ParentChildIndexFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/ParentChildIndexFieldData.java index 647e275664c53..381a7ae9435fe 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/ParentChildIndexFieldData.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/plain/ParentChildIndexFieldData.java @@ -278,6 +278,11 @@ public int ordValue() { public boolean advanceExact(int target) throws IOException { return segmentValues.advanceExact(target); } + + @Override + public int docID() { + return segmentValues.docID(); + } }; } diff --git a/core/src/main/java/org/elasticsearch/search/MultiValueMode.java b/core/src/main/java/org/elasticsearch/search/MultiValueMode.java index c75cbb3c10a4e..b6ffa7f1591a9 100644 --- a/core/src/main/java/org/elasticsearch/search/MultiValueMode.java +++ b/core/src/main/java/org/elasticsearch/search/MultiValueMode.java @@ -777,6 +777,11 @@ public boolean advanceExact(int target) throws IOException { return values.advanceExact(target); } + @Override + public int docID() { + return values.docID(); + } + @Override public int ordValue() { try { @@ -857,6 +862,11 @@ public boolean advanceExact(int rootDoc) throws IOException { return true; } + @Override + public int docID() { + return lastSeenRootDoc; + } + @Override public int ordValue() { return lastEmittedOrd; diff --git a/core/src/test/java/org/elasticsearch/search/MultiValueModeTests.java b/core/src/test/java/org/elasticsearch/search/MultiValueModeTests.java index d33e0543f5cba..210a2217e3454 100644 --- a/core/src/test/java/org/elasticsearch/search/MultiValueModeTests.java +++ b/core/src/test/java/org/elasticsearch/search/MultiValueModeTests.java @@ -462,6 +462,7 @@ public BytesRef nextValue() { @Override public boolean advanceExact(int doc) { this.doc = doc; + i = 0; return array[doc].length > 0; } @@ -566,6 +567,12 @@ public boolean advanceExact(int target) throws IOException { docID = target; return array[docID] != -1; } + + @Override + public int docID() { + return docID; + } + @Override public int ordValue() { return array[docID]; From 870452f9f4daeb2b8d6cc30b9c7a854e737a2f4c Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Tue, 11 Apr 2017 14:23:24 +0200 Subject: [PATCH 51/87] Initialize doc IDs to the correct value. --- .../elasticsearch/index/fielddata/ordinals/MultiOrdinals.java | 2 +- .../index/fielddata/ordinals/SinglePackedOrdinals.java | 2 +- .../index/fielddata/plain/IndexIndexFieldData.java | 2 +- .../test/java/org/elasticsearch/search/MultiValueModeTests.java | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/MultiOrdinals.java b/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/MultiOrdinals.java index 642d6e0a78760..86e2787658c35 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/MultiOrdinals.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/MultiOrdinals.java @@ -116,7 +116,7 @@ private static class SingleDocs extends AbstractSortedDocValues { private final PackedLongValues ords; private final ValuesHolder values; - private int currentDoc; + private int currentDoc = -1; private long currentStartOffset; private long currentEndOffset; diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/SinglePackedOrdinals.java b/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/SinglePackedOrdinals.java index daedae519b0da..a7d93693e89fa 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/SinglePackedOrdinals.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/SinglePackedOrdinals.java @@ -68,7 +68,7 @@ private static class Docs extends AbstractSortedDocValues { private final PackedInts.Reader reader; private final ValuesHolder values; - private int currentDoc; + private int currentDoc = -1; private int currentOrd; Docs(SinglePackedOrdinals parent, ValuesHolder values) { diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/IndexIndexFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/IndexIndexFieldData.java index 1b9f07bf8a5ee..53832c0b5b153 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/IndexIndexFieldData.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/plain/IndexIndexFieldData.java @@ -81,7 +81,7 @@ public SortedSetDocValues getOrdinalsValues() { final BytesRef term = new BytesRef(index); final SortedDocValues sortedValues = new AbstractSortedDocValues() { - private int docID; + private int docID = -1; @Override public BytesRef lookupOrd(int ord) { diff --git a/core/src/test/java/org/elasticsearch/search/MultiValueModeTests.java b/core/src/test/java/org/elasticsearch/search/MultiValueModeTests.java index 210a2217e3454..6b8a4023b5e30 100644 --- a/core/src/test/java/org/elasticsearch/search/MultiValueModeTests.java +++ b/core/src/test/java/org/elasticsearch/search/MultiValueModeTests.java @@ -561,7 +561,7 @@ public void testSingleValuedOrds() throws Exception { } } final SortedDocValues singleValues = new AbstractSortedDocValues() { - private int docID; + private int docID = -1; @Override public boolean advanceExact(int target) throws IOException { docID = target; From 7b82e5db8ab5dd8c7eaa9f87ea4558dcadf29359 Mon Sep 17 00:00:00 2001 From: Colin Goodheart-Smithe Date: Tue, 11 Apr 2017 13:36:32 +0100 Subject: [PATCH 52/87] Fixes MissingValuesTests --- .../aggregations/support/MissingValues.java | 4 ++-- .../support/MissingValuesTests.java | 24 ++++++++++--------- 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/support/MissingValues.java b/core/src/main/java/org/elasticsearch/search/aggregations/support/MissingValues.java index a34860ee4496c..0d68b5fd6a563 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/support/MissingValues.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/support/MissingValues.java @@ -241,7 +241,7 @@ public long nextOrd() throws IOException { @Override public boolean advanceExact(int doc) throws IOException { hasOrds = values.advanceExact(doc); - nextMissingOrd = SortedSetDocValues.NO_MORE_ORDS; + nextMissingOrd = missingOrd; // always return true because we want to return a value even if // the document does not have a value return true; @@ -294,7 +294,7 @@ public long nextOrd() throws IOException { @Override public boolean advanceExact(int doc) throws IOException { hasOrds = values.advanceExact(doc); - nextMissingOrd = SortedSetDocValues.NO_MORE_ORDS; + nextMissingOrd = insertedOrd; // always return true because we want to return a value even if // the document does not have a value return true; diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/support/MissingValuesTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/support/MissingValuesTests.java index e29c59dd2328b..568b8e7996f1e 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/support/MissingValuesTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/support/MissingValuesTests.java @@ -58,13 +58,13 @@ public void testMissingBytes() throws IOException { @Override public BytesRef nextValue() { - return values[doc][doc++]; + return values[doc][i++]; } @Override public boolean advanceExact(int docId) { doc = docId; - doc = 0; + i = 0; return values[doc].length > 0; } @@ -119,8 +119,8 @@ public void testMissingOrds() throws IOException { @Override public boolean advanceExact(int docID) { doc = docID; - doc = 0; - return values[doc].length > 0; + i = 0; + return ords[doc].length > 0; } @Override @@ -135,8 +135,8 @@ public long getValueCount() { @Override public long nextOrd() { - if (doc < ords[doc].length) { - return ords[doc][doc++]; + if (i < ords[doc].length) { + return ords[doc][i++]; } else { return NO_MORE_ORDS; } @@ -156,8 +156,9 @@ public long nextOrd() { for (int i = 0; i < numDocs; ++i) { assertTrue(withMissingReplaced.advanceExact(i)); if (ords[i].length > 0) { - for (long ord : ords[i]) { - assertEquals(ord, withMissingReplaced.nextOrd()); + for (int ord : ords[i]) { + assertEquals(values[ord], + withMissingReplaced.lookupOrd(withMissingReplaced.nextOrd())); } assertEquals(SortedSetDocValues.NO_MORE_ORDS, withMissingReplaced.nextOrd()); } else { @@ -185,13 +186,13 @@ public void testMissingLongs() throws IOException { @Override public long nextValue() { - return values[doc][doc++]; + return values[doc][i++]; } @Override public boolean advanceExact(int docId) { doc = docId; - doc = 0; + i = 0; return values[doc].length > 0; } @@ -233,12 +234,13 @@ public void testMissingDoubles() throws IOException { @Override public double nextValue() { - return values[doc][doc++]; + return values[doc][i++]; } @Override public boolean advanceExact(int docId) { doc = docId; + i = 0; return true; } From d86a2875698ecf9155edee2efa41d79c1dfada78 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Tue, 11 Apr 2017 14:40:11 +0200 Subject: [PATCH 53/87] Implement docID() on NumericDocValues. --- .../grouping/CollapsingDocValuesSource.java | 5 +++++ .../fielddata/AbstractNumericDocValues.java | 5 ----- .../SortableLongBitsNumericDocValues.java | 7 +++++++ .../elasticsearch/search/MultiValueMode.java | 20 ++++++++++++++++++- ...DiversifiedBytesHashSamplerAggregator.java | 8 ++++++++ .../DiversifiedMapSamplerAggregator.java | 8 ++++++++ .../DiversifiedNumericSamplerAggregator.java | 5 +++++ .../DiversifiedOrdinalsSamplerAggregator.java | 10 ++++++++++ .../index/fielddata/FieldDataTests.java | 6 ++++++ .../search/MultiValueModeTests.java | 6 +++++- 10 files changed, 73 insertions(+), 7 deletions(-) diff --git a/core/src/main/java/org/apache/lucene/search/grouping/CollapsingDocValuesSource.java b/core/src/main/java/org/apache/lucene/search/grouping/CollapsingDocValuesSource.java index 09d79ee28f173..2f805e08c2d6d 100644 --- a/core/src/main/java/org/apache/lucene/search/grouping/CollapsingDocValuesSource.java +++ b/core/src/main/java/org/apache/lucene/search/grouping/CollapsingDocValuesSource.java @@ -108,6 +108,11 @@ public boolean advanceExact(int target) throws IOException { } } + @Override + public int docID() { + return sorted.docID(); + } + @Override public long longValue() throws IOException { return value; diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/AbstractNumericDocValues.java b/core/src/main/java/org/elasticsearch/index/fielddata/AbstractNumericDocValues.java index effefa3bdb3e4..21c96f3090241 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/AbstractNumericDocValues.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/AbstractNumericDocValues.java @@ -32,11 +32,6 @@ */ public abstract class AbstractNumericDocValues extends NumericDocValues { - @Override - public int docID() { - throw new UnsupportedOperationException(); - } - @Override public int nextDoc() throws IOException { throw new UnsupportedOperationException(); diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/SortableLongBitsNumericDocValues.java b/core/src/main/java/org/elasticsearch/index/fielddata/SortableLongBitsNumericDocValues.java index af1b273d1c142..259070cfc92aa 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/SortableLongBitsNumericDocValues.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/SortableLongBitsNumericDocValues.java @@ -31,6 +31,7 @@ */ final class SortableLongBitsNumericDocValues extends AbstractNumericDocValues { + private int docID = -1; private final NumericDoubleValues values; SortableLongBitsNumericDocValues(NumericDoubleValues values) { @@ -44,9 +45,15 @@ public long longValue() throws IOException { @Override public boolean advanceExact(int target) throws IOException { + docID = target; return values.advanceExact(target); } + @Override + public int docID() { + return docID; + } + /** Return the wrapped values. */ public NumericDoubleValues getDoubleValues() { return values; diff --git a/core/src/main/java/org/elasticsearch/search/MultiValueMode.java b/core/src/main/java/org/elasticsearch/search/MultiValueMode.java index b6ffa7f1591a9..efe7f621ccdf9 100644 --- a/core/src/main/java/org/elasticsearch/search/MultiValueMode.java +++ b/core/src/main/java/org/elasticsearch/search/MultiValueMode.java @@ -464,6 +464,11 @@ public boolean advanceExact(int target) throws IOException { return true; } + @Override + public int docID() { + return singleton.docID(); + } + @Override public long longValue() throws IOException { return hasValue ? singleton.longValue() : missingValue; @@ -480,6 +485,11 @@ public boolean advanceExact(int target) throws IOException { return true; } + @Override + public int docID() { + return values.docID(); + } + @Override public long longValue() throws IOException { return hasValue ? pick(values) : missingValue; @@ -511,7 +521,7 @@ public NumericDocValues select(final SortedNumericDocValues values, final long m return new AbstractNumericDocValues() { - int lastSeenRootDoc = 0; + int lastSeenRootDoc = -1; long lastEmittedValue = missingValue; @Override @@ -520,6 +530,9 @@ public boolean advanceExact(int rootDoc) throws IOException { assert rootDoc >= lastSeenRootDoc : "can only evaluate current and upcoming root docs"; if (rootDoc == lastSeenRootDoc) { return true; + } else if (rootDoc == 0) { + lastEmittedValue = missingValue; + return true; } final int prevRootDoc = rootDocs.prevSetBit(rootDoc - 1); final int firstNestedDoc; @@ -534,6 +547,11 @@ public boolean advanceExact(int rootDoc) throws IOException { return true; } + @Override + public int docID() { + return lastSeenRootDoc; + } + @Override public long longValue() { return lastEmittedValue; diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedBytesHashSamplerAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedBytesHashSamplerAggregator.java index 0bc678d25e4e8..472881b0084eb 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedBytesHashSamplerAggregator.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedBytesHashSamplerAggregator.java @@ -100,8 +100,11 @@ protected NumericDocValues getKeys(LeafReaderContext context) { } return new AbstractNumericDocValues() { + private int docID = -1; + @Override public boolean advanceExact(int target) throws IOException { + docID = target; if (values.advanceExact(target)) { if (values.docValueCount() > 1) { throw new IllegalArgumentException( @@ -113,6 +116,11 @@ public boolean advanceExact(int target) throws IOException { } } + @Override + public int docID() { + return docID; + } + @Override public long longValue() throws IOException { final BytesRef bytes = values.nextValue(); diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedMapSamplerAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedMapSamplerAggregator.java index 0480e013c10fe..935da11e132b1 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedMapSamplerAggregator.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedMapSamplerAggregator.java @@ -106,8 +106,11 @@ protected NumericDocValues getKeys(LeafReaderContext context) { } return new AbstractNumericDocValues() { + private int docID = -1; + @Override public boolean advanceExact(int target) throws IOException { + docID = target; if (values.advanceExact(target)) { if (values.docValueCount() > 1) { throw new IllegalArgumentException( @@ -119,6 +122,11 @@ public boolean advanceExact(int target) throws IOException { } } + @Override + public int docID() { + return docID; + } + @Override public long longValue() throws IOException { final BytesRef bytes = values.nextValue(); diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedNumericSamplerAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedNumericSamplerAggregator.java index 4badde70b5b1d..8ce29c14bb5ee 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedNumericSamplerAggregator.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedNumericSamplerAggregator.java @@ -104,6 +104,11 @@ public boolean advanceExact(int target) throws IOException { } } + @Override + public int docID() { + return values.docID(); + } + @Override public long longValue() throws IOException { return values.nextValue(); diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedOrdinalsSamplerAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedOrdinalsSamplerAggregator.java index cda371c6e734b..74f7a16d6216d 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedOrdinalsSamplerAggregator.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedOrdinalsSamplerAggregator.java @@ -95,6 +95,11 @@ public boolean advanceExact(int target) throws IOException { return singleValues.advanceExact(target); } + @Override + public int docID() { + return singleValues.docID(); + } + @Override public long longValue() throws IOException { return singleValues.ordValue(); @@ -121,6 +126,11 @@ public boolean advanceExact(int target) throws IOException { } } + @Override + public int docID() { + return globalOrds.docID(); + } + @Override public long longValue() throws IOException { return value; diff --git a/core/src/test/java/org/elasticsearch/index/fielddata/FieldDataTests.java b/core/src/test/java/org/elasticsearch/index/fielddata/FieldDataTests.java index 0c01d4064d38d..af9059d8b9b26 100644 --- a/core/src/test/java/org/elasticsearch/index/fielddata/FieldDataTests.java +++ b/core/src/test/java/org/elasticsearch/index/fielddata/FieldDataTests.java @@ -34,11 +34,17 @@ public void testSortableLongBitsToDoubles() throws IOException { final long valueBits = NumericUtils.doubleToSortableLong(value); NumericDocValues values = new AbstractNumericDocValues() { + private int docID = -1; @Override public boolean advanceExact(int target) throws IOException { + docID = target; return true; } @Override + public int docID() { + return docID; + } + @Override public long longValue() throws IOException { return valueBits; } diff --git a/core/src/test/java/org/elasticsearch/search/MultiValueModeTests.java b/core/src/test/java/org/elasticsearch/search/MultiValueModeTests.java index 6b8a4023b5e30..c6023a09461a0 100644 --- a/core/src/test/java/org/elasticsearch/search/MultiValueModeTests.java +++ b/core/src/test/java/org/elasticsearch/search/MultiValueModeTests.java @@ -87,13 +87,17 @@ public void testSingleValuedLongs() throws Exception { } } final NumericDocValues singleValues = new AbstractNumericDocValues() { - int docId; + int docId = -1; @Override public boolean advanceExact(int target) throws IOException { this.docId = target; return docsWithValue == null ? true : docsWithValue.get(docId); } @Override + public int docID() { + return docId; + } + @Override public long longValue() { return array[docId]; } From ea1eb00dfa36796b72ce162893845933c3f6f538 Mon Sep 17 00:00:00 2001 From: Colin Goodheart-Smithe Date: Tue, 11 Apr 2017 14:13:38 +0100 Subject: [PATCH 54/87] Fixes tests --- .../index/codec/CodecService.java | 6 ++-- .../aggregations/support/MissingValues.java | 28 +++++++++++++------ 2 files changed, 23 insertions(+), 11 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/index/codec/CodecService.java b/core/src/main/java/org/elasticsearch/index/codec/CodecService.java index 59be64a85d623..30342848974fd 100644 --- a/core/src/main/java/org/elasticsearch/index/codec/CodecService.java +++ b/core/src/main/java/org/elasticsearch/index/codec/CodecService.java @@ -22,7 +22,7 @@ import org.apache.logging.log4j.Logger; import org.apache.lucene.codecs.Codec; import org.apache.lucene.codecs.lucene50.Lucene50StoredFieldsFormat.Mode; -import org.apache.lucene.codecs.lucene62.Lucene62Codec; +import org.apache.lucene.codecs.lucene70.Lucene70Codec; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.collect.MapBuilder; import org.elasticsearch.index.mapper.MapperService; @@ -47,8 +47,8 @@ public class CodecService { public CodecService(@Nullable MapperService mapperService, Logger logger) { final MapBuilder codecs = MapBuilder.newMapBuilder(); if (mapperService == null) { - codecs.put(DEFAULT_CODEC, new Lucene62Codec()); - codecs.put(BEST_COMPRESSION_CODEC, new Lucene62Codec(Mode.BEST_COMPRESSION)); + codecs.put(DEFAULT_CODEC, new Lucene70Codec()); + codecs.put(BEST_COMPRESSION_CODEC, new Lucene70Codec(Mode.BEST_COMPRESSION)); } else { codecs.put(DEFAULT_CODEC, new PerFieldMappingPostingFormatCodec(Mode.BEST_SPEED, mapperService, logger)); diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/support/MissingValues.java b/core/src/main/java/org/elasticsearch/search/aggregations/support/MissingValues.java index 0d68b5fd6a563..fdd3701a13098 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/support/MissingValues.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/support/MissingValues.java @@ -58,8 +58,11 @@ static SortedBinaryDocValues replaceMissing(final SortedBinaryDocValues values, @Override public boolean advanceExact(int doc) throws IOException { - values.advanceExact(doc); - count = values.docValueCount(); + if (values.advanceExact(doc)) { + count = values.docValueCount(); + } else { + count = 0; + } // always return true because we want to return a value even if // the document does not have a value return true; @@ -131,8 +134,11 @@ public int docValueCount() { @Override public boolean advanceExact(int doc) throws IOException { - values.advanceExact(doc); - count = values.docValueCount(); + if (values.advanceExact(doc)) { + count = values.docValueCount(); + } else { + count = 0; + } // always return true because we want to return a value even if // the document does not have a value return true; @@ -148,8 +154,11 @@ static SortedNumericDoubleValues replaceMissing(final SortedNumericDoubleValues @Override public boolean advanceExact(int doc) throws IOException { - values.advanceExact(doc); - count = values.docValueCount(); + if (values.advanceExact(doc)) { + count = values.docValueCount(); + } else { + count = 0; + } // always return true because we want to return a value even if // the document does not have a value return true; @@ -325,8 +334,11 @@ static MultiGeoPointValues replaceMissing(final MultiGeoPointValues values, fina @Override public boolean advanceExact(int doc) throws IOException { - values.advanceExact(doc); - count = values.docValueCount(); + if (values.advanceExact(doc)) { + count = values.docValueCount(); + } else { + count = 0; + } // always return true because we want to return a value even if // the document does not have a value return true; From 063790488c0043d2e752b2b195294719045196cd Mon Sep 17 00:00:00 2001 From: Colin Goodheart-Smithe Date: Tue, 11 Apr 2017 14:27:16 +0100 Subject: [PATCH 55/87] Fixes Geo tests --- .../index/fielddata/plain/LatLonPointDVAtomicFieldData.java | 1 + 1 file changed, 1 insertion(+) diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/LatLonPointDVAtomicFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/LatLonPointDVAtomicFieldData.java index 99cdbc451ae2b..cf6d6a2a26b07 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/LatLonPointDVAtomicFieldData.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/plain/LatLonPointDVAtomicFieldData.java @@ -63,6 +63,7 @@ public MultiGeoPointValues getGeoPointValues() { @Override public boolean advanceExact(int doc) throws IOException { + pointsCursor = 0; if (values.advanceExact(doc)) { count = values.docValueCount(); if (count > points.length) { From 838d73e2b8b55417bb3d0ac2e3581855795103f9 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Tue, 11 Apr 2017 15:50:50 +0200 Subject: [PATCH 56/87] Fix failures in MultiValueModeTests. --- .../elasticsearch/search/MultiValueMode.java | 16 ++- .../search/MultiValueModeTests.java | 101 ++++++++++-------- 2 files changed, 69 insertions(+), 48 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/search/MultiValueMode.java b/core/src/main/java/org/elasticsearch/search/MultiValueMode.java index efe7f621ccdf9..a033e193b78e7 100644 --- a/core/src/main/java/org/elasticsearch/search/MultiValueMode.java +++ b/core/src/main/java/org/elasticsearch/search/MultiValueMode.java @@ -675,6 +675,9 @@ protected double pick(SortedNumericDoubleValues values, double missingValue, Doc public BinaryDocValues select(final SortedBinaryDocValues values, final BytesRef missingValue) { final BinaryDocValues singleton = FieldData.unwrapSingleton(values); if (singleton != null) { + if (missingValue == null) { + return singleton; + } return new AbstractBinaryDocValues() { private boolean hasValue; @@ -693,17 +696,22 @@ public BytesRef binaryValue() throws IOException { } else { return new AbstractBinaryDocValues() { - private boolean hasValue; + private BytesRef value; @Override public boolean advanceExact(int target) throws IOException { - hasValue = values.advanceExact(target); - return true; + if (values.advanceExact(target)) { + value = pick(values); + return true; + } else { + value = missingValue; + return missingValue != null; + } } @Override public BytesRef binaryValue() throws IOException { - return hasValue ? pick(values) : missingValue; + return value; } }; } diff --git a/core/src/test/java/org/elasticsearch/search/MultiValueModeTests.java b/core/src/test/java/org/elasticsearch/search/MultiValueModeTests.java index c6023a09461a0..c9994171d6494 100644 --- a/core/src/test/java/org/elasticsearch/search/MultiValueModeTests.java +++ b/core/src/test/java/org/elasticsearch/search/MultiValueModeTests.java @@ -41,6 +41,7 @@ import org.elasticsearch.index.fielddata.NumericDoubleValues; import org.elasticsearch.index.fielddata.SortedBinaryDocValues; import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; +import org.elasticsearch.search.MultiValueMode.UnsortedNumericDoubleValues; import org.elasticsearch.test.ESTestCase; import java.io.IOException; @@ -50,6 +51,11 @@ public class MultiValueModeTests extends ESTestCase { + @FunctionalInterface + private static interface Supplier { + T get() throws IOException; + } + private static FixedBitSet randomRootDocs(int maxDoc) { FixedBitSet set = new FixedBitSet(maxDoc); for (int i = 0; i < maxDoc; ++i) { @@ -86,7 +92,8 @@ public void testSingleValuedLongs() throws Exception { docsWithValue.set(i); } } - final NumericDocValues singleValues = new AbstractNumericDocValues() { + + final Supplier multiValues = () -> DocValues.singleton(new AbstractNumericDocValues() { int docId = -1; @Override public boolean advanceExact(int target) throws IOException { @@ -101,12 +108,11 @@ public int docID() { public long longValue() { return array[docId]; } - }; - final SortedNumericDocValues multiValues = DocValues.singleton(singleValues); - verify(multiValues, numDocs); + }); + verifySortedNumeric(multiValues, numDocs); final FixedBitSet rootDocs = randomRootDocs(numDocs); final FixedBitSet innerDocs = randomInnerDocs(rootDocs); - verify(multiValues, numDocs, rootDocs, innerDocs); + verifySortedNumeric(multiValues, numDocs, rootDocs, innerDocs); } public void testMultiValuedLongs() throws Exception { @@ -120,7 +126,7 @@ public void testMultiValuedLongs() throws Exception { Arrays.sort(values); array[i] = values; } - final SortedNumericDocValues multiValues = new AbstractSortedNumericDocValues() { + final Supplier multiValues = () -> new AbstractSortedNumericDocValues() { int doc; int i; @@ -141,15 +147,16 @@ public int docValueCount() { return array[doc].length; } }; - verify(multiValues, numDocs); + verifySortedNumeric(multiValues, numDocs); final FixedBitSet rootDocs = randomRootDocs(numDocs); final FixedBitSet innerDocs = randomInnerDocs(rootDocs); - verify(multiValues, numDocs, rootDocs, innerDocs); + verifySortedNumeric(multiValues, numDocs, rootDocs, innerDocs); } - private void verify(SortedNumericDocValues values, int maxDoc) throws IOException { + private void verifySortedNumeric(Supplier supplier, int maxDoc) throws IOException { for (long missingValue : new long[] { 0, randomLong() }) { for (MultiValueMode mode : MultiValueMode.values()) { + SortedNumericDocValues values = supplier.get(); final NumericDocValues selected = mode.select(values, missingValue); for (int i = 0; i < maxDoc; ++i) { assertTrue(selected.advanceExact(i)); @@ -181,7 +188,7 @@ private void verify(SortedNumericDocValues values, int maxDoc) throws IOExceptio for (int j = 0; j < value - 1; ++j) { values.nextValue(); } - expected = Math.round((values.nextValue() + values.nextValue())/2.0); + expected = Math.round(((double) values.nextValue() + values.nextValue())/2.0); } else { for (int j = 0; j < value; ++j) { values.nextValue(); @@ -197,9 +204,10 @@ private void verify(SortedNumericDocValues values, int maxDoc) throws IOExceptio } } - private void verify(SortedNumericDocValues values, int maxDoc, FixedBitSet rootDocs, FixedBitSet innerDocs) throws IOException { + private void verifySortedNumeric(Supplier supplier, int maxDoc, FixedBitSet rootDocs, FixedBitSet innerDocs) throws IOException { for (long missingValue : new long[] { 0, randomLong() }) { for (MultiValueMode mode : new MultiValueMode[] {MultiValueMode.MIN, MultiValueMode.MAX, MultiValueMode.SUM, MultiValueMode.AVG}) { + SortedNumericDocValues values = supplier.get(); final NumericDocValues selected = mode.select(values, missingValue, rootDocs, new BitSetIterator(innerDocs, 0L), maxDoc); int prevRoot = -1; for (int root = rootDocs.nextSetBit(0); root != -1; root = root + 1 < maxDoc ? rootDocs.nextSetBit(root + 1) : -1) { @@ -254,7 +262,7 @@ public void testSingleValuedDoubles() throws Exception { docsWithValue.set(i); } } - final NumericDoubleValues singleValues = new NumericDoubleValues() { + final Supplier multiValues = () -> FieldData.singleton(new NumericDoubleValues() { int docID; @Override public boolean advanceExact(int doc) throws IOException { @@ -265,12 +273,11 @@ public boolean advanceExact(int doc) throws IOException { public double doubleValue() { return array[docID]; } - }; - final SortedNumericDoubleValues multiValues = FieldData.singleton(singleValues); - verify(multiValues, numDocs); + }); + verifySortedNumericDouble(multiValues, numDocs); final FixedBitSet rootDocs = randomRootDocs(numDocs); final FixedBitSet innerDocs = randomInnerDocs(rootDocs); - verify(multiValues, numDocs, rootDocs, innerDocs); + verifySortedNumericDouble(multiValues, numDocs, rootDocs, innerDocs); } public void testMultiValuedDoubles() throws Exception { @@ -284,7 +291,7 @@ public void testMultiValuedDoubles() throws Exception { Arrays.sort(values); array[i] = values; } - final SortedNumericDoubleValues multiValues = new SortedNumericDoubleValues() { + final Supplier multiValues = () -> new SortedNumericDoubleValues() { int doc; int i; @@ -305,18 +312,19 @@ public int docValueCount() { return array[doc].length; } }; - verify(multiValues, numDocs); + verifySortedNumericDouble(multiValues, numDocs); final FixedBitSet rootDocs = randomRootDocs(numDocs); final FixedBitSet innerDocs = randomInnerDocs(rootDocs); - verify(multiValues, numDocs, rootDocs, innerDocs); + verifySortedNumericDouble(multiValues, numDocs, rootDocs, innerDocs); } - private void verify(SortedNumericDoubleValues values, int maxDoc) throws IOException { + private void verifySortedNumericDouble(Supplier supplier, int maxDoc) throws IOException { for (long missingValue : new long[] { 0, randomLong() }) { for (MultiValueMode mode : MultiValueMode.values()) { if (MultiValueMode.MEDIAN.equals(mode)) { continue; } + SortedNumericDoubleValues values = supplier.get(); final NumericDoubleValues selected = mode.select(values, missingValue); for (int i = 0; i < maxDoc; ++i) { assertTrue(selected.advanceExact(i)); @@ -364,9 +372,10 @@ private void verify(SortedNumericDoubleValues values, int maxDoc) throws IOExcep } } - private void verify(SortedNumericDoubleValues values, int maxDoc, FixedBitSet rootDocs, FixedBitSet innerDocs) throws IOException { + private void verifySortedNumericDouble(Supplier supplier, int maxDoc, FixedBitSet rootDocs, FixedBitSet innerDocs) throws IOException { for (long missingValue : new long[] { 0, randomLong() }) { for (MultiValueMode mode : new MultiValueMode[] {MultiValueMode.MIN, MultiValueMode.MAX, MultiValueMode.SUM, MultiValueMode.AVG}) { + SortedNumericDoubleValues values = supplier.get(); final NumericDoubleValues selected = mode.select(values, missingValue, rootDocs, new BitSetIterator(innerDocs, 0L), maxDoc); int prevRoot = -1; for (int root = rootDocs.nextSetBit(0); root != -1; root = root + 1 < maxDoc ? rootDocs.nextSetBit(root + 1) : -1) { @@ -424,7 +433,7 @@ public void testSingleValuedStrings() throws Exception { } } } - final BinaryDocValues singleValues = new AbstractBinaryDocValues() { + final Supplier multiValues = () -> FieldData.singleton(new AbstractBinaryDocValues() { int docID; @Override public boolean advanceExact(int target) throws IOException { @@ -435,12 +444,11 @@ public boolean advanceExact(int target) throws IOException { public BytesRef binaryValue() { return BytesRef.deepCopyOf(array[docID]); } - }; - final SortedBinaryDocValues multiValues = FieldData.singleton(singleValues); - verify(multiValues, numDocs); + }); + verifySortedBinary(multiValues, numDocs); final FixedBitSet rootDocs = randomRootDocs(numDocs); final FixedBitSet innerDocs = randomInnerDocs(rootDocs); - verify(multiValues, numDocs, rootDocs, innerDocs); + verifySortedBinary(multiValues, numDocs, rootDocs, innerDocs); } public void testMultiValuedStrings() throws Exception { @@ -454,7 +462,7 @@ public void testMultiValuedStrings() throws Exception { Arrays.sort(values); array[i] = values; } - final SortedBinaryDocValues multiValues = new SortedBinaryDocValues() { + final Supplier multiValues = () -> new SortedBinaryDocValues() { int doc; int i; @@ -475,15 +483,16 @@ public int docValueCount() { return array[doc].length; } }; - verify(multiValues, numDocs); + verifySortedBinary(multiValues, numDocs); final FixedBitSet rootDocs = randomRootDocs(numDocs); final FixedBitSet innerDocs = randomInnerDocs(rootDocs); - verify(multiValues, numDocs, rootDocs, innerDocs); + verifySortedBinary(multiValues, numDocs, rootDocs, innerDocs); } - private void verify(SortedBinaryDocValues values, int maxDoc) throws IOException { + private void verifySortedBinary(Supplier supplier, int maxDoc) throws IOException { for (BytesRef missingValue : new BytesRef[] { new BytesRef(), new BytesRef(RandomStrings.randomAsciiOfLength(random(), 8)) }) { for (MultiValueMode mode : new MultiValueMode[] {MultiValueMode.MIN, MultiValueMode.MAX}) { + SortedBinaryDocValues values = supplier.get(); final BinaryDocValues selected = mode.select(values, missingValue); for (int i = 0; i < maxDoc; ++i) { assertTrue(selected.advanceExact(i)); @@ -516,9 +525,10 @@ private void verify(SortedBinaryDocValues values, int maxDoc) throws IOException } } - private void verify(SortedBinaryDocValues values, int maxDoc, FixedBitSet rootDocs, FixedBitSet innerDocs) throws IOException { + private void verifySortedBinary(Supplier supplier, int maxDoc, FixedBitSet rootDocs, FixedBitSet innerDocs) throws IOException { for (BytesRef missingValue : new BytesRef[] { new BytesRef(), new BytesRef(RandomStrings.randomAsciiOfLength(random(), 8)) }) { for (MultiValueMode mode : new MultiValueMode[] {MultiValueMode.MIN, MultiValueMode.MAX}) { + SortedBinaryDocValues values = supplier.get(); final BinaryDocValues selected = mode.select(values, missingValue, rootDocs, new BitSetIterator(innerDocs, 0L), maxDoc); int prevRoot = -1; for (int root = rootDocs.nextSetBit(0); root != -1; root = root + 1 < maxDoc ? rootDocs.nextSetBit(root + 1) : -1) { @@ -564,7 +574,7 @@ public void testSingleValuedOrds() throws Exception { array[i] = -1; } } - final SortedDocValues singleValues = new AbstractSortedDocValues() { + final Supplier multiValues = () -> DocValues.singleton(new AbstractSortedDocValues() { private int docID = -1; @Override public boolean advanceExact(int target) throws IOException { @@ -591,12 +601,11 @@ public BytesRef lookupOrd(int ord) { public int getValueCount() { return 1 << 20; } - }; - final SortedSetDocValues multiValues = DocValues.singleton(singleValues); - verify(multiValues, numDocs); + }); + verifySortedSet(multiValues, numDocs); final FixedBitSet rootDocs = randomRootDocs(numDocs); final FixedBitSet innerDocs = randomInnerDocs(rootDocs); - verify(multiValues, numDocs, rootDocs, innerDocs); + verifySortedSet(multiValues, numDocs, rootDocs, innerDocs); } public void testMultiValuedOrds() throws Exception { @@ -609,7 +618,7 @@ public void testMultiValuedOrds() throws Exception { } array[i] = values; } - final SortedSetDocValues multiValues = new AbstractSortedSetDocValues() { + final Supplier multiValues = () -> new AbstractSortedSetDocValues() { int doc; int i; @@ -625,6 +634,7 @@ public long nextOrd() { @Override public boolean advanceExact(int docID) { this.doc = docID; + i = 0; return array[doc].length > 0; } @@ -638,14 +648,15 @@ public long getValueCount() { return 1 << 20; } }; - verify(multiValues, numDocs); + verifySortedSet(multiValues, numDocs); final FixedBitSet rootDocs = randomRootDocs(numDocs); final FixedBitSet innerDocs = randomInnerDocs(rootDocs); - verify(multiValues, numDocs, rootDocs, innerDocs); + verifySortedSet(multiValues, numDocs, rootDocs, innerDocs); } - private void verify(SortedSetDocValues values, int maxDoc) throws IOException { + private void verifySortedSet(Supplier supplier, int maxDoc) throws IOException { for (MultiValueMode mode : new MultiValueMode[] {MultiValueMode.MIN, MultiValueMode.MAX}) { + SortedSetDocValues values = supplier.get(); final SortedDocValues selected = mode.select(values); for (int i = 0; i < maxDoc; ++i) { final long actual = selected.advanceExact(i) ? selected.ordValue() : -1; @@ -669,8 +680,9 @@ private void verify(SortedSetDocValues values, int maxDoc) throws IOException { } } - private void verify(SortedSetDocValues values, int maxDoc, FixedBitSet rootDocs, FixedBitSet innerDocs) throws IOException { + private void verifySortedSet(Supplier supplier, int maxDoc, FixedBitSet rootDocs, FixedBitSet innerDocs) throws IOException { for (MultiValueMode mode : new MultiValueMode[] {MultiValueMode.MIN, MultiValueMode.MAX}) { + SortedSetDocValues values = supplier.get(); final SortedDocValues selected = mode.select(values, rootDocs, new BitSetIterator(innerDocs, 0L)); int prevRoot = -1; for (int root = rootDocs.nextSetBit(0); root != -1; root = root + 1 < maxDoc ? rootDocs.nextSetBit(root + 1) : -1) { @@ -743,7 +755,7 @@ public double nextValue() throws IOException { return Math.cos(singletonValues.nextValue()); } }; - verify(multiValues, numDocs); + verifyUnsortedNumeric(() -> multiValues, numDocs); } public void testUnsortedMultiValuedDoubles() throws Exception { @@ -778,12 +790,13 @@ public double nextValue() { return Math.sin(array[doc][i++]); } }; - verify(multiValues, numDocs); + verifyUnsortedNumeric(() -> multiValues, numDocs); } - private void verify(MultiValueMode.UnsortedNumericDoubleValues values, int maxDoc) throws IOException { + private void verifyUnsortedNumeric(Supplier supplier, int maxDoc) throws IOException { for (double missingValue : new double[] { 0, randomDouble() }) { for (MultiValueMode mode : new MultiValueMode[] {MultiValueMode.MIN, MultiValueMode.MAX, MultiValueMode.SUM, MultiValueMode.AVG}) { + UnsortedNumericDoubleValues values = supplier.get(); final NumericDoubleValues selected = mode.select(values, missingValue); for (int i = 0; i < maxDoc; ++i) { assertTrue(selected.advanceExact(i)); From 4d9c1718ff4c26b7ad8e0769ee3d23cd65274c4e Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Tue, 11 Apr 2017 15:53:07 +0200 Subject: [PATCH 57/87] Fix failure of ParentFieldSubFetchPhase. --- .../search/fetch/subphase/ParentFieldSubFetchPhase.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/org/elasticsearch/search/fetch/subphase/ParentFieldSubFetchPhase.java b/core/src/main/java/org/elasticsearch/search/fetch/subphase/ParentFieldSubFetchPhase.java index 7a0a530e621be..0ffef32e42790 100644 --- a/core/src/main/java/org/elasticsearch/search/fetch/subphase/ParentFieldSubFetchPhase.java +++ b/core/src/main/java/org/elasticsearch/search/fetch/subphase/ParentFieldSubFetchPhase.java @@ -62,7 +62,7 @@ public void hitExecute(SearchContext context, HitContext hitContext) { public static String getParentId(ParentFieldMapper fieldMapper, LeafReader reader, int docId) { try { SortedDocValues docValues = reader.getSortedDocValues(fieldMapper.name()); - if (docValues == null || docValues.advanceExact(docId)) { + if (docValues == null || docValues.advanceExact(docId) == false) { // hit has no _parent field. return null; } From fbe413d53aceea4e5e8f5fe3adaca6f97597f35d Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Tue, 11 Apr 2017 15:54:13 +0200 Subject: [PATCH 58/87] Fix NestedQueryBuilderTests failures. --- .../org/elasticsearch/index/query/NestedQueryBuilderTests.java | 1 - 1 file changed, 1 deletion(-) diff --git a/core/src/test/java/org/elasticsearch/index/query/NestedQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/NestedQueryBuilderTests.java index d8cdf221ca14f..267963878ea16 100644 --- a/core/src/test/java/org/elasticsearch/index/query/NestedQueryBuilderTests.java +++ b/core/src/test/java/org/elasticsearch/index/query/NestedQueryBuilderTests.java @@ -152,7 +152,6 @@ public void testFromJson() throws IOException { " }\n" + " }\n" + " } ],\n" + - " \"disable_coord\" : false,\n" + " \"adjust_pure_negative\" : true,\n" + " \"boost\" : 1.0\n" + " }\n" + From 12c31fa19f837ff32bb4e0ecef2af6339c174271 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Tue, 11 Apr 2017 15:55:36 +0200 Subject: [PATCH 59/87] Fix failures in QueryStringQueryBuilderTests. --- .../apache/lucene/queryparser/classic/MapperQueryParser.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/org/apache/lucene/queryparser/classic/MapperQueryParser.java b/core/src/main/java/org/apache/lucene/queryparser/classic/MapperQueryParser.java index 8298f723bf746..79f522e8c1fe5 100644 --- a/core/src/main/java/org/apache/lucene/queryparser/classic/MapperQueryParser.java +++ b/core/src/main/java/org/apache/lucene/queryparser/classic/MapperQueryParser.java @@ -101,13 +101,13 @@ public void reset(QueryParserSettings settings) { setAnalyzer(settings.analyzer()); setMultiTermRewriteMethod(settings.rewriteMethod()); setEnablePositionIncrements(settings.enablePositionIncrements()); + setSplitOnWhitespace(settings.splitOnWhitespace()); setAutoGeneratePhraseQueries(settings.autoGeneratePhraseQueries()); setMaxDeterminizedStates(settings.maxDeterminizedStates()); setAllowLeadingWildcard(settings.allowLeadingWildcard()); setPhraseSlop(settings.phraseSlop()); setDefaultOperator(settings.defaultOperator()); setFuzzyPrefixLength(settings.fuzzyPrefixLength()); - setSplitOnWhitespace(settings.splitOnWhitespace()); } /** From 103315e6da76f0fa2c5c35a3a3643a303cebbb8f Mon Sep 17 00:00:00 2001 From: Colin Goodheart-Smithe Date: Tue, 11 Apr 2017 14:57:12 +0100 Subject: [PATCH 60/87] More tests fixes --- .../elasticsearch/index/fielddata/ScriptDocValues.java | 8 +++++--- .../index/fielddata/ScriptDocValuesGeoPointsTests.java | 1 + .../index/fielddata/ScriptDocValuesLongsTests.java | 1 + 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java b/core/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java index 5776f84a83ff5..0a87c334244c0 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java @@ -155,6 +155,7 @@ public static final class Longs extends ScriptDocValues { private long[] values = new long[0]; private int count; private Dates dates; + private int docId; public Longs(SortedNumericDocValues in) { this.in = in; @@ -164,6 +165,7 @@ public Longs(SortedNumericDocValues in) { @Override public void setNextDocId(int docId) throws IOException { if (in.advanceExact(docId)) { + this.docId = docId; resize(in.docValueCount()); for (int i = 0; i < count; i++) { values[i] = in.nextValue(); @@ -172,7 +174,7 @@ public void setNextDocId(int docId) throws IOException { resize(0); } if (dates != null) { - dates.refreshArray(); + dates.setNextDocId(docId); } } @@ -201,7 +203,7 @@ public ReadableDateTime getDate() throws IOException { deprecationLogger.deprecated("getDate on numeric fields is deprecated. Use a date field to get dates."); if (dates == null) { dates = new Dates(in); - dates.refreshArray(); + dates.setNextDocId(docId); } return dates.getValue(); } @@ -211,7 +213,7 @@ public List getDates() throws IOException { deprecationLogger.deprecated("getDates on numeric fields is deprecated. Use a date field to get dates."); if (dates == null) { dates = new Dates(in); - dates.refreshArray(); + dates.setNextDocId(docId); } return dates; } diff --git a/core/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesGeoPointsTests.java b/core/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesGeoPointsTests.java index dab348f89d619..1f71a808ab58a 100644 --- a/core/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesGeoPointsTests.java +++ b/core/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesGeoPointsTests.java @@ -43,6 +43,7 @@ public GeoPoint nextValue() { @Override public boolean advanceExact(int docId) { + docID = docId; return points.length > 0; } diff --git a/core/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesLongsTests.java b/core/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesLongsTests.java index 3a560a4f629f5..1b3e8fa227462 100644 --- a/core/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesLongsTests.java +++ b/core/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesLongsTests.java @@ -94,6 +94,7 @@ private Longs wrap(long[][] values) { @Override public boolean advanceExact(int doc) { + i = 0; current = values[doc]; return current.length > 0; } From 504554c14ce6ff752badf189829e6ea325c770e7 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Tue, 11 Apr 2017 16:09:02 +0200 Subject: [PATCH 61/87] Remove redundant static modifier. --- .../test/java/org/elasticsearch/search/MultiValueModeTests.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/test/java/org/elasticsearch/search/MultiValueModeTests.java b/core/src/test/java/org/elasticsearch/search/MultiValueModeTests.java index c9994171d6494..1a357c55eb056 100644 --- a/core/src/test/java/org/elasticsearch/search/MultiValueModeTests.java +++ b/core/src/test/java/org/elasticsearch/search/MultiValueModeTests.java @@ -52,7 +52,7 @@ public class MultiValueModeTests extends ESTestCase { @FunctionalInterface - private static interface Supplier { + private interface Supplier { T get() throws IOException; } From d145ba67b176f7e3ee4c1c4c515f314fe6468435 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Tue, 11 Apr 2017 17:47:22 +0200 Subject: [PATCH 62/87] More test failures. --- .../BytesRefFieldComparatorSource.java | 11 +++++--- .../elasticsearch/search/MultiValueMode.java | 27 +++++++++++-------- .../AbstractFieldDataImplTestCase.java | 1 + .../fielddata/BinaryDVFieldDataTests.java | 2 +- 4 files changed, 25 insertions(+), 16 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/BytesRefFieldComparatorSource.java b/core/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/BytesRefFieldComparatorSource.java index 4be5282e79286..2e697e0408804 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/BytesRefFieldComparatorSource.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/BytesRefFieldComparatorSource.java @@ -154,6 +154,7 @@ static class ReplaceMissing extends AbstractSortedDocValues { final int substituteOrd; final BytesRef substituteTerm; final boolean exists; + boolean hasValue; ReplaceMissing(SortedDocValues in, BytesRef term) throws IOException { this.in = in; @@ -170,10 +171,11 @@ static class ReplaceMissing extends AbstractSortedDocValues { @Override public int ordValue() { - int ord = in.ordValue(); - if (ord < 0) { + if (hasValue == false) { return substituteOrd; - } else if (exists == false && ord >= substituteOrd) { + } + int ord = in.ordValue(); + if (exists == false && ord >= substituteOrd) { return ord + 1; } else { return ord; @@ -182,7 +184,8 @@ public int ordValue() { @Override public boolean advanceExact(int target) throws IOException { - return in.advanceExact(target); + hasValue = in.advanceExact(target); + return true; } @Override diff --git a/core/src/main/java/org/elasticsearch/search/MultiValueMode.java b/core/src/main/java/org/elasticsearch/search/MultiValueMode.java index a033e193b78e7..acccb6d71d684 100644 --- a/core/src/main/java/org/elasticsearch/search/MultiValueMode.java +++ b/core/src/main/java/org/elasticsearch/search/MultiValueMode.java @@ -798,9 +798,17 @@ public SortedDocValues select(final SortedSetDocValues values) { } else { return new AbstractSortedDocValues() { + int ord; + @Override public boolean advanceExact(int target) throws IOException { - return values.advanceExact(target); + if (values.advanceExact(target)) { + ord = pick(values); + return true; + } else { + ord = -1; + return false; + } } @Override @@ -810,12 +818,8 @@ public int docID() { @Override public int ordValue() { - try { - return pick(values); - } catch (IOException e) { - // https://issues.apache.org/jira/browse/LUCENE-7767 - throw new RuntimeException(e); - } + assert ord != -1; + return ord; } @Override @@ -854,6 +858,7 @@ public SortedDocValues select(final SortedSetDocValues values, final BitSet root return new AbstractSortedDocValues() { + int docID = -1; int lastSeenRootDoc = 0; int lastEmittedOrd = -1; @@ -872,7 +877,7 @@ public boolean advanceExact(int rootDoc) throws IOException { assert rootDocs.get(rootDoc) : "can only sort root documents"; assert rootDoc >= lastSeenRootDoc : "can only evaluate current and upcoming root docs"; if (rootDoc == lastSeenRootDoc) { - return true; + return lastEmittedOrd != -1; } final int prevRootDoc = rootDocs.prevSetBit(rootDoc - 1); @@ -883,14 +888,14 @@ public boolean advanceExact(int rootDoc) throws IOException { firstNestedDoc = innerDocs.advance(prevRootDoc + 1); } - lastSeenRootDoc = rootDoc; + docID = lastSeenRootDoc = rootDoc; lastEmittedOrd = pick(selectedValues, innerDocs, firstNestedDoc, rootDoc); - return true; + return lastEmittedOrd != -1; } @Override public int docID() { - return lastSeenRootDoc; + return docID; } @Override diff --git a/core/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataImplTestCase.java b/core/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataImplTestCase.java index 6973d3b26703e..c22114e28aafd 100644 --- a/core/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataImplTestCase.java +++ b/core/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataImplTestCase.java @@ -104,6 +104,7 @@ public void testSingleValueAllSet() throws Exception { assertThat(bytesValues.docValueCount(), equalTo(1)); assertThat(bytesValues.nextValue(), equalTo(new BytesRef(three()))); + bytesValues = fieldData.getBytesValues(); assertValues(bytesValues, 0, two()); assertValues(bytesValues, 1, one()); assertValues(bytesValues, 2, three()); diff --git a/core/src/test/java/org/elasticsearch/index/fielddata/BinaryDVFieldDataTests.java b/core/src/test/java/org/elasticsearch/index/fielddata/BinaryDVFieldDataTests.java index 64ab07442b4c2..dfb3a3c1b3ec2 100644 --- a/core/src/test/java/org/elasticsearch/index/fielddata/BinaryDVFieldDataTests.java +++ b/core/src/test/java/org/elasticsearch/index/fielddata/BinaryDVFieldDataTests.java @@ -91,7 +91,7 @@ public void testDocValue() throws Exception { assertThat(bytesValues.nextValue(), equalTo(new BytesRef(bytesList1.get(0)))); assertThat(bytesValues.nextValue(), equalTo(new BytesRef(bytesList1.get(1)))); - assertTrue(bytesValues.advanceExact(0)); + assertTrue(bytesValues.advanceExact(1)); assertThat(bytesValues.docValueCount(), equalTo(1)); assertThat(bytesValues.nextValue(), equalTo(new BytesRef(bytes1))); From fd5d2da3247c2b252c4a8ec9ece224ab61f25463 Mon Sep 17 00:00:00 2001 From: Colin Goodheart-Smithe Date: Tue, 11 Apr 2017 15:26:48 +0100 Subject: [PATCH 63/87] more test fixes --- .../java/org/elasticsearch/deps/lucene/SimpleLuceneTests.java | 2 ++ .../index/fielddata/ordinals/MultiOrdinalsTests.java | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/core/src/test/java/org/elasticsearch/deps/lucene/SimpleLuceneTests.java b/core/src/test/java/org/elasticsearch/deps/lucene/SimpleLuceneTests.java index 086a504505973..6b117fc7e96dc 100644 --- a/core/src/test/java/org/elasticsearch/deps/lucene/SimpleLuceneTests.java +++ b/core/src/test/java/org/elasticsearch/deps/lucene/SimpleLuceneTests.java @@ -23,6 +23,7 @@ import org.apache.lucene.document.Field; import org.apache.lucene.document.IntPoint; import org.apache.lucene.document.SortedDocValuesField; +import org.apache.lucene.document.StoredField; import org.apache.lucene.document.TextField; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.FieldInfo; @@ -79,6 +80,7 @@ public void testSimpleNumericOps() throws Exception { Document document = new Document(); document.add(new TextField("_id", "1", Field.Store.YES)); document.add(new IntPoint("test", 2)); + document.add(new StoredField("test", 2)); indexWriter.addDocument(document); IndexReader reader = DirectoryReader.open(indexWriter); diff --git a/core/src/test/java/org/elasticsearch/index/fielddata/ordinals/MultiOrdinalsTests.java b/core/src/test/java/org/elasticsearch/index/fielddata/ordinals/MultiOrdinalsTests.java index 304940288d6a4..1ae6197c547da 100644 --- a/core/src/test/java/org/elasticsearch/index/fielddata/ordinals/MultiOrdinalsTests.java +++ b/core/src/test/java/org/elasticsearch/index/fielddata/ordinals/MultiOrdinalsTests.java @@ -264,7 +264,7 @@ private void assertEquals(SortedSetDocValues docs, long[][] ordinalPlan) throws assertThat(FieldData.isMultiValued(docs), equalTo(true)); for (int doc = 0; doc < ordinalPlan.length; ++doc) { long[] ords = ordinalPlan[doc]; - assertEquals(ords.length > 0, docs.advance(doc)); + assertEquals(ords.length > 0, docs.advanceExact(doc)); if (ords.length > 0) { for (long ord : ords) { assertThat(docs.nextOrd(), equalTo(ord)); From 9be08c1a8a5f5a01f2aee69fe436409731bcf532 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Tue, 11 Apr 2017 18:25:13 +0200 Subject: [PATCH 64/87] Fix remaining errors in the fielddata package. --- .../BytesRefFieldComparatorSource.java | 11 +---- .../AbstractAtomicParentChildFieldData.java | 14 +++++- .../index/fielddata/FieldDataTests.java | 45 ++++++++++++------- 3 files changed, 43 insertions(+), 27 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/BytesRefFieldComparatorSource.java b/core/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/BytesRefFieldComparatorSource.java index 2e697e0408804..87cbb7d86c4e8 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/BytesRefFieldComparatorSource.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/BytesRefFieldComparatorSource.java @@ -113,8 +113,6 @@ public void setScorer(Scorer scorer) { }; } - final BytesRef nullPlaceHolder = new BytesRef(); - final BytesRef nonNullMissingBytes = missingBytes == null ? nullPlaceHolder : missingBytes; return new FieldComparator.TermValComparator(numHits, null, sortMissingLast) { @Override @@ -122,20 +120,15 @@ protected BinaryDocValues getBinaryDocValues(LeafReaderContext context, String f final SortedBinaryDocValues values = getValues(context); final BinaryDocValues selectedValues; if (nested == null) { - selectedValues = sortMode.select(values, nonNullMissingBytes); + selectedValues = sortMode.select(values, missingBytes); } else { final BitSet rootDocs = nested.rootDocs(context); final DocIdSetIterator innerDocs = nested.innerDocs(context); - selectedValues = sortMode.select(values, nonNullMissingBytes, rootDocs, innerDocs, context.reader().maxDoc()); + selectedValues = sortMode.select(values, missingBytes, rootDocs, innerDocs, context.reader().maxDoc()); } return selectedValues; } - @Override - protected boolean isNull(int doc, BytesRef term) { - return term == nullPlaceHolder; - } - @Override public void setScorer(Scorer scorer) { BytesRefFieldComparatorSource.this.setScorer(scorer); diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractAtomicParentChildFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractAtomicParentChildFieldData.java index f82d91d37b6ce..2df5aa6bb63fd 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractAtomicParentChildFieldData.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractAtomicParentChildFieldData.java @@ -47,16 +47,26 @@ public final ScriptDocValues getScriptValues() { public final SortedBinaryDocValues getBytesValues() { return new SortedBinaryDocValues() { + private final SortedDocValues[] perTypeValues; private final BytesRef[] terms = new BytesRef[2]; private int count; private int termsCursor; + { + Set types = types(); + perTypeValues = new SortedDocValues[types.size()]; + int i = 0; + for (String type : types) { + perTypeValues[i++] = getOrdinalsValues(type); + } + } + @Override public boolean advanceExact(int docId) throws IOException { count = 0; termsCursor = 0; - for (String type : types()) { - final SortedDocValues values = getOrdinalsValues(type); + + for (SortedDocValues values : perTypeValues) { if (values.advanceExact(docId)) { final int ord = values.ordValue(); terms[count++] = values.lookupOrd(ord); diff --git a/core/src/test/java/org/elasticsearch/index/fielddata/FieldDataTests.java b/core/src/test/java/org/elasticsearch/index/fielddata/FieldDataTests.java index af9059d8b9b26..6236517dde0be 100644 --- a/core/src/test/java/org/elasticsearch/index/fielddata/FieldDataTests.java +++ b/core/src/test/java/org/elasticsearch/index/fielddata/FieldDataTests.java @@ -29,26 +29,37 @@ public class FieldDataTests extends ESTestCase { + private static class DummyValues extends AbstractNumericDocValues { + + private final long value; + private int docID = -1; + + DummyValues(long value) { + this.value = value; + } + + @Override + public boolean advanceExact(int target) throws IOException { + docID = target; + return true; + } + + @Override + public int docID() { + return docID; + } + + @Override + public long longValue() throws IOException { + return value; + } + } + public void testSortableLongBitsToDoubles() throws IOException { final double value = randomDouble(); final long valueBits = NumericUtils.doubleToSortableLong(value); - NumericDocValues values = new AbstractNumericDocValues() { - private int docID = -1; - @Override - public boolean advanceExact(int target) throws IOException { - docID = target; - return true; - } - @Override - public int docID() { - return docID; - } - @Override - public long longValue() throws IOException { - return valueBits; - } - }; + NumericDocValues values = new DummyValues(valueBits); SortedNumericDoubleValues asMultiDoubles = FieldData.sortableLongBitsToDoubles(DocValues.singleton(values)); NumericDoubleValues asDoubles = FieldData.unwrapSingleton(asMultiDoubles); @@ -56,6 +67,8 @@ public long longValue() throws IOException { assertTrue(asDoubles.advanceExact(0)); assertEquals(value, asDoubles.doubleValue(), 0); + values = new DummyValues(valueBits); + asMultiDoubles = FieldData.sortableLongBitsToDoubles(DocValues.singleton(values)); NumericDocValues backToLongs = DocValues.unwrapSingleton(FieldData.toSortableLongBits(asMultiDoubles)); assertSame(values, backToLongs); From 6bf6e57aecf828d95a29d9e1f6e8639d26ef0a67 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Wed, 12 Apr 2017 08:52:27 +0200 Subject: [PATCH 65/87] Fix more test failures. --- .../indices/IndicesRequestCache.java | 3 +++ .../support/values/ScriptDoubleValues.java | 16 ++++++++++---- .../support/values/ScriptLongValues.java | 16 ++++++++++---- .../AnalysisFactoryTestCase.java | 2 ++ .../test/FieldMaskingReader.java | 21 ++++++++++++++++++- 5 files changed, 49 insertions(+), 9 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/indices/IndicesRequestCache.java b/core/src/main/java/org/elasticsearch/indices/IndicesRequestCache.java index d941b097a6010..bd6d75ea3d6dc 100644 --- a/core/src/main/java/org/elasticsearch/indices/IndicesRequestCache.java +++ b/core/src/main/java/org/elasticsearch/indices/IndicesRequestCache.java @@ -265,6 +265,9 @@ public void onClose(IndexReader.CacheKey cacheKey) { @Override public boolean equals(Object o) { if (this == o) return true; + if (o == null || getClass() != o.getClass()) { + return false; + } CleanupKey that = (CleanupKey) o; if (readerVersion != that.readerVersion) return false; if (!entity.getCacheIdentity().equals(that.entity.getCacheIdentity())) return false; diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/support/values/ScriptDoubleValues.java b/core/src/main/java/org/elasticsearch/search/aggregations/support/values/ScriptDoubleValues.java index 673e09a4f430c..619ffde0a1e8d 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/support/values/ScriptDoubleValues.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/support/values/ScriptDoubleValues.java @@ -55,14 +55,22 @@ public boolean advanceExact(int target) throws IOException { resize(1); values[0] = ((ReadableInstant) value).getMillis(); } else if (value.getClass().isArray()) { - resize(Array.getLength(value)); - for (int i = 0; i < docValueCount(); ++i) { + int length = Array.getLength(value); + if (length == 0) { + return false; + } + resize(length); + for (int i = 0; i < length; ++i) { values[i] = toDoubleValue(Array.get(value, i)); } } else if (value instanceof Collection) { - resize(((Collection) value).size()); + Collection coll = (Collection) value; + if (coll.isEmpty()) { + return false; + } + resize(coll.size()); int i = 0; - for (Object v : (Collection) value) { + for (Object v : coll) { values[i++] = toDoubleValue(v); } assert i == docValueCount(); diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/support/values/ScriptLongValues.java b/core/src/main/java/org/elasticsearch/search/aggregations/support/values/ScriptLongValues.java index 042d987b6e433..6247e96c7ec11 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/support/values/ScriptLongValues.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/support/values/ScriptLongValues.java @@ -53,16 +53,24 @@ public boolean advanceExact(int target) throws IOException { } else if (value.getClass().isArray()) { - resize(Array.getLength(value)); - for (int i = 0; i < docValueCount(); ++i) { + int length = Array.getLength(value); + if (length == 0) { + return false; + } + resize(length); + for (int i = 0; i < length; ++i) { values[i] = toLongValue(Array.get(value, i)); } } else if (value instanceof Collection) { - resize(((Collection) value).size()); + Collection coll = (Collection) value; + if (coll.isEmpty()) { + return false; + } + resize(coll.size()); int i = 0; - for (Iterator it = ((Collection) value).iterator(); it.hasNext(); ++i) { + for (Iterator it = coll.iterator(); it.hasNext(); ++i) { values[i] = toLongValue(it.next()); } assert i == docValueCount(); diff --git a/test/framework/src/main/java/org/elasticsearch/AnalysisFactoryTestCase.java b/test/framework/src/main/java/org/elasticsearch/AnalysisFactoryTestCase.java index 76d33a6c42565..83f955296b706 100644 --- a/test/framework/src/main/java/org/elasticsearch/AnalysisFactoryTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/AnalysisFactoryTestCase.java @@ -285,6 +285,8 @@ private static String toCamelCase(String s) { .put("fingerprint", Void.class) // for tee-sinks .put("daterecognizer", Void.class) + // for token filters that generate bad offsets, which are now rejected since Lucene 7 + .put("fixbrokenoffsets", Void.class) .immutableMap(); diff --git a/test/framework/src/main/java/org/elasticsearch/test/FieldMaskingReader.java b/test/framework/src/main/java/org/elasticsearch/test/FieldMaskingReader.java index 4a7c95cfd44f8..3aba68868f746 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/FieldMaskingReader.java +++ b/test/framework/src/main/java/org/elasticsearch/test/FieldMaskingReader.java @@ -21,7 +21,9 @@ import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.FieldFilterLeafReader; import org.apache.lucene.index.FilterDirectoryReader; +import org.apache.lucene.index.FilterLeafReader; import org.apache.lucene.index.LeafReader; +import org.apache.lucene.index.IndexReader.CacheHelper; import java.io.IOException; import java.util.Collections; @@ -32,7 +34,24 @@ public FieldMaskingReader(String field, DirectoryReader in) throws IOException { super(in, new FilterDirectoryReader.SubReaderWrapper() { @Override public LeafReader wrap(LeafReader reader) { - return new FieldFilterLeafReader(reader, Collections.singleton(field), true); + return new FilterLeafReader(new FieldFilterLeafReader(reader, Collections.singleton(field), true)) { + + // FieldFilterLeafReader does not forward cache helpers + // since it considers it is illegal because of the fact + // that it changes the content of the index. However we + // want this behavior for tests, and security plugins + // are careful to only use the cache when it's valid + + @Override + public CacheHelper getReaderCacheHelper() { + return reader.getReaderCacheHelper(); + } + + @Override + public CacheHelper getCoreCacheHelper() { + return reader.getCoreCacheHelper(); + } + }; } }); this.field = field; From a07cae32379e640c29e55fa176e69f656f39816a Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Wed, 12 Apr 2017 09:05:37 +0200 Subject: [PATCH 66/87] Fix last failures in ReplaceMissingTests. --- .../index/fielddata/fieldcomparator/ReplaceMissingTests.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/core/src/test/java/org/elasticsearch/index/fielddata/fieldcomparator/ReplaceMissingTests.java b/core/src/test/java/org/elasticsearch/index/fielddata/fieldcomparator/ReplaceMissingTests.java index f897fc70861cc..65c6335ebbc2d 100644 --- a/core/src/test/java/org/elasticsearch/index/fielddata/fieldcomparator/ReplaceMissingTests.java +++ b/core/src/test/java/org/elasticsearch/index/fielddata/fieldcomparator/ReplaceMissingTests.java @@ -69,6 +69,7 @@ public void test() throws Exception { assertTrue(dv.advanceExact(2)); assertEquals(1, dv.ordValue()); + raw = ar.getSortedDocValues("field"); dv = new BytesRefFieldComparatorSource.ReplaceMissing(raw, new BytesRef("dog")); assertEquals(2, dv.getValueCount()); assertEquals("cat", dv.lookupOrd(0).utf8ToString()); @@ -82,6 +83,7 @@ public void test() throws Exception { assertEquals(1, dv.ordValue()); // non-existing values + raw = ar.getSortedDocValues("field"); dv = new BytesRefFieldComparatorSource.ReplaceMissing(raw, new BytesRef("apple")); assertEquals(3, dv.getValueCount()); assertEquals("apple", dv.lookupOrd(0).utf8ToString()); @@ -95,6 +97,7 @@ public void test() throws Exception { assertTrue(dv.advanceExact(2)); assertEquals(2, dv.ordValue()); + raw = ar.getSortedDocValues("field"); dv = new BytesRefFieldComparatorSource.ReplaceMissing(raw, new BytesRef("company")); assertEquals(3, dv.getValueCount()); assertEquals("cat", dv.lookupOrd(0).utf8ToString()); @@ -108,6 +111,7 @@ public void test() throws Exception { assertTrue(dv.advanceExact(2)); assertEquals(2, dv.ordValue()); + raw = ar.getSortedDocValues("field"); dv = new BytesRefFieldComparatorSource.ReplaceMissing(raw, new BytesRef("ebay")); assertEquals(3, dv.getValueCount()); assertEquals("cat", dv.lookupOrd(0).utf8ToString()); From 965c73240d76d21f087ca1b4cf2e8610425dacff Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Wed, 12 Apr 2017 10:05:04 +0200 Subject: [PATCH 67/87] Add AwaitsFix. --- .../org/elasticsearch/common/lucene/ShardCoreKeyMapTests.java | 1 + 1 file changed, 1 insertion(+) diff --git a/core/src/test/java/org/elasticsearch/common/lucene/ShardCoreKeyMapTests.java b/core/src/test/java/org/elasticsearch/common/lucene/ShardCoreKeyMapTests.java index ca24a1346fe11..515d62140d2e6 100644 --- a/core/src/test/java/org/elasticsearch/common/lucene/ShardCoreKeyMapTests.java +++ b/core/src/test/java/org/elasticsearch/common/lucene/ShardCoreKeyMapTests.java @@ -57,6 +57,7 @@ public void testMissingShard() throws IOException { } } + @AwaitsFix(bugUrl="https://issues.apache.org/jira/browse/LUCENE-7781") public void testAddingAClosedReader() throws Exception { LeafReader reader; try (Directory dir = newDirectory(); From 974f44f1e7de349180ffe5f237bbdc4cac7c7d1a Mon Sep 17 00:00:00 2001 From: Colin Goodheart-Smithe Date: Wed, 12 Apr 2017 09:15:36 +0100 Subject: [PATCH 68/87] Fixes ScriptValuesTests --- .../aggregations/support/values/ScriptBytesValues.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/support/values/ScriptBytesValues.java b/core/src/main/java/org/elasticsearch/search/aggregations/support/values/ScriptBytesValues.java index 9b98097923dfe..aee70b859d0a6 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/support/values/ScriptBytesValues.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/support/values/ScriptBytesValues.java @@ -56,12 +56,18 @@ public boolean advanceExact(int doc) throws IOException { return false; } else if (value.getClass().isArray()) { count = Array.getLength(value); + if (count == 0) { + return false; + } grow(); for (int i = 0; i < count; ++i) { set(i, Array.get(value, i)); } } else if (value instanceof Collection) { final Collection coll = (Collection) value; + if (coll.isEmpty()) { + return false; + } count = coll.size(); grow(); int i = 0; From 80d0631201dff20c47850c1a442c477c6ba415c7 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Wed, 12 Apr 2017 10:50:17 +0200 Subject: [PATCH 69/87] Fix more integ test failures. --- .../index/fielddata/ScriptDocValues.java | 17 ++++++++++++----- .../cardinality/CardinalityAggregator.java | 5 ++--- .../support/values/ScriptBytesValues.java | 3 +++ 3 files changed, 17 insertions(+), 8 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java b/core/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java index 0a87c334244c0..d8e33da6b99a3 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java @@ -23,6 +23,7 @@ import org.apache.lucene.index.SortedNumericDocValues; import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.BytesRefBuilder; import org.elasticsearch.common.geo.GeoHashUtils; import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.common.geo.GeoUtils; @@ -87,7 +88,7 @@ public final void sort(Comparator c) { public static final class Strings extends ScriptDocValues { private final SortedBinaryDocValues in; - private BytesRef[] values = new BytesRef[0]; + private BytesRefBuilder[] values = new BytesRefBuilder[0]; private int count; public Strings(SortedBinaryDocValues in) { @@ -99,7 +100,7 @@ public void setNextDocId(int docId) throws IOException { if (in.advanceExact(docId)) { resize(in.docValueCount()); for (int i = 0; i < count; i++) { - values[i] = in.nextValue(); + values[i].copyBytes(in.nextValue()); } } else { resize(0); @@ -112,7 +113,13 @@ public void setNextDocId(int docId) throws IOException { */ protected void resize(int newSize) { count = newSize; - values = ArrayUtil.grow(values, count); + if (newSize > values.length) { + final int oldLength = values.length; + values = ArrayUtil.grow(values, count); + for (int i = oldLength; i < values.length; ++i) { + values[i] = new BytesRefBuilder(); + } + } } public SortedBinaryDocValues getInternalValues() { @@ -121,7 +128,7 @@ public SortedBinaryDocValues getInternalValues() { public BytesRef getBytesValue() { if (size() > 0) { - return values[0]; + return values[0].get(); } else { return null; } @@ -138,7 +145,7 @@ public String getValue() { @Override public String get(int index) { - return values[index].utf8ToString(); + return values[index].get().utf8ToString(); } @Override diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/CardinalityAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/CardinalityAggregator.java index b982590e5234a..7a8483b1b26ee 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/CardinalityAggregator.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/CardinalityAggregator.java @@ -251,9 +251,8 @@ public void collect(int doc, long bucketOrd) throws IOException { visitedOrds.set(bucketOrd, bits); } if (values.advanceExact(doc)) { - final long valueCount = values.getValueCount(); - for (int i = 0; i < valueCount; ++i) { - bits.set((int) values.nextOrd()); + for (long ord = values.nextOrd(); ord != SortedSetDocValues.NO_MORE_ORDS; ord = values.nextOrd()) { + bits.set((int) ord); } } } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/support/values/ScriptBytesValues.java b/core/src/main/java/org/elasticsearch/search/aggregations/support/values/ScriptBytesValues.java index aee70b859d0a6..9e0b3b9ef6d3b 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/support/values/ScriptBytesValues.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/support/values/ScriptBytesValues.java @@ -69,6 +69,9 @@ public boolean advanceExact(int doc) throws IOException { return false; } count = coll.size(); + if (count == 0) { + return false; + } grow(); int i = 0; for (Object v : coll) { From 8cf4b18eba7ac24935d302d332e5d1e4311454d9 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Wed, 12 Apr 2017 11:25:01 +0200 Subject: [PATCH 70/87] More fixes. --- .../index/fielddata/FieldData.java | 3 +- .../index/fielddata/MultiGeoPointValues.java | 2 +- .../AbstractLatLonPointDVIndexFieldData.java | 18 ++--- .../plain/LatLonPointDVAtomicFieldData.java | 70 ++++++++----------- .../functionscore/DecayFunctionBuilder.java | 2 +- .../geocentroid/InternalGeoCentroid.java | 4 +- .../aggregations/support/MissingValues.java | 2 +- 7 files changed, 41 insertions(+), 60 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/FieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/FieldData.java index 268a8198d0aa3..b39e6cb049eca 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/FieldData.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/FieldData.java @@ -26,7 +26,6 @@ import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.NumericUtils; import org.elasticsearch.common.geo.GeoPoint; import java.io.IOException; @@ -461,7 +460,7 @@ public boolean advanceExact(int doc) throws IOException { return values.advanceExact(doc); } @Override - public void get(List list) { + public void get(List list) throws IOException { for (int i = 0, count = values.docValueCount(); i < count; ++i) { list.add(values.nextValue().toString()); } diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/MultiGeoPointValues.java b/core/src/main/java/org/elasticsearch/index/fielddata/MultiGeoPointValues.java index 82bcd8acb6702..c80c337c6d0e3 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/MultiGeoPointValues.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/MultiGeoPointValues.java @@ -64,6 +64,6 @@ protected MultiGeoPointValues() { * * @return the next value for the current docID set to {@link #advanceExact(int)}. */ - public abstract GeoPoint nextValue(); + public abstract GeoPoint nextValue() throws IOException; } diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractLatLonPointDVIndexFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractLatLonPointDVIndexFieldData.java index 3b4ac58e0e81e..6c92d571196ec 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractLatLonPointDVIndexFieldData.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractLatLonPointDVIndexFieldData.java @@ -19,13 +19,11 @@ package org.elasticsearch.index.fielddata.plain; import org.apache.lucene.document.LatLonDocValuesField; -import org.apache.lucene.index.DocValues; import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.search.SortField; -import org.elasticsearch.ElasticsearchException; import org.elasticsearch.common.Nullable; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexSettings; @@ -38,8 +36,6 @@ import org.elasticsearch.indices.breaker.CircuitBreakerService; import org.elasticsearch.search.MultiValueMode; -import java.io.IOException; - public abstract class AbstractLatLonPointDVIndexFieldData extends DocValuesIndexFieldData implements IndexGeoPointFieldData { AbstractLatLonPointDVIndexFieldData(Index index, String fieldName) { @@ -58,16 +54,12 @@ public LatLonPointDVIndexFieldData(Index index, String fieldName) { @Override public AtomicGeoPointFieldData load(LeafReaderContext context) { - try { - LeafReader reader = context.reader(); - FieldInfo info = reader.getFieldInfos().fieldInfo(fieldName); - if (info != null) { - checkCompatible(info); - } - return new LatLonPointDVAtomicFieldData(DocValues.getSortedNumeric(reader, fieldName)); - } catch (IOException e) { - throw new IllegalStateException("Cannot load doc values", e); + LeafReader reader = context.reader(); + FieldInfo info = reader.getFieldInfos().fieldInfo(fieldName); + if (info != null) { + checkCompatible(info); } + return new LatLonPointDVAtomicFieldData(reader, fieldName); } @Override diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/LatLonPointDVAtomicFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/LatLonPointDVAtomicFieldData.java index cf6d6a2a26b07..93fa5bf22acc7 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/LatLonPointDVAtomicFieldData.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/plain/LatLonPointDVAtomicFieldData.java @@ -19,6 +19,8 @@ package org.elasticsearch.index.fielddata.plain; import org.apache.lucene.geo.GeoEncodingUtils; +import org.apache.lucene.index.DocValues; +import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.SortedNumericDocValues; import org.apache.lucene.util.Accountable; import org.apache.lucene.util.ArrayUtil; @@ -27,16 +29,17 @@ import org.elasticsearch.index.fielddata.MultiGeoPointValues; import java.io.IOException; -import java.util.Arrays; import java.util.Collection; import java.util.Collections; final class LatLonPointDVAtomicFieldData extends AbstractAtomicGeoPointFieldData { - private final SortedNumericDocValues values; + private final LeafReader reader; + private final String fieldName; - LatLonPointDVAtomicFieldData(SortedNumericDocValues values) { + LatLonPointDVAtomicFieldData(LeafReader reader, String fieldName) { super(); - this.values = values; + this.reader = reader; + this.fieldName = fieldName; } @Override @@ -56,45 +59,32 @@ public void close() { @Override public MultiGeoPointValues getGeoPointValues() { - return new MultiGeoPointValues() { - GeoPoint[] points = new GeoPoint[0]; - private int pointsCursor; - private int count = 0; + try { + final SortedNumericDocValues numericValues = DocValues.getSortedNumeric(reader, fieldName); + return new MultiGeoPointValues() { - @Override - public boolean advanceExact(int doc) throws IOException { - pointsCursor = 0; - if (values.advanceExact(doc)) { - count = values.docValueCount(); - if (count > points.length) { - final int previousLength = points.length; - points = Arrays.copyOf(points, - ArrayUtil.oversize(count, RamUsageEstimator.NUM_BYTES_OBJECT_REF)); - for (int i = previousLength; i < points.length; ++i) { - points[i] = new GeoPoint(Double.NaN, Double.NaN); - } - } - long encoded; - for (int i = 0; i < count; ++i) { - encoded = values.nextValue(); - points[i].reset(GeoEncodingUtils.decodeLatitude((int) (encoded >>> 32)), - GeoEncodingUtils.decodeLongitude((int) encoded)); - } - return true; - } else { - return false; + final GeoPoint point = new GeoPoint(); + + @Override + public boolean advanceExact(int doc) throws IOException { + return numericValues.advanceExact(doc); } - } - @Override - public int docValueCount() { - return count; - } + @Override + public int docValueCount() { + return numericValues.docValueCount(); + } - @Override - public GeoPoint nextValue() { - return points[pointsCursor++]; - } - }; + @Override + public GeoPoint nextValue() throws IOException { + final long encoded = numericValues.nextValue(); + point.reset(GeoEncodingUtils.decodeLatitude((int) (encoded >>> 32)), + GeoEncodingUtils.decodeLongitude((int) encoded)); + return point; + } + }; + } catch (IOException e) { + throw new IllegalStateException("Cannot load doc values", e); + } } } diff --git a/core/src/main/java/org/elasticsearch/index/query/functionscore/DecayFunctionBuilder.java b/core/src/main/java/org/elasticsearch/index/query/functionscore/DecayFunctionBuilder.java index 92b4e592165a6..e296c102188b5 100644 --- a/core/src/main/java/org/elasticsearch/index/query/functionscore/DecayFunctionBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/query/functionscore/DecayFunctionBuilder.java @@ -358,7 +358,7 @@ public boolean advanceExact(int docId) throws IOException { } @Override - public double nextValue() { + public double nextValue() throws IOException { GeoPoint other = geoPointValues.nextValue(); return Math.max(0.0d, distFunction.calculate(origin.lat(), origin.lon(), other.lat(), other.lon(), DistanceUnit.METERS) - offset); diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/InternalGeoCentroid.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/InternalGeoCentroid.java index f97423f3bd09b..63ca5d27df2d8 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/InternalGeoCentroid.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/InternalGeoCentroid.java @@ -39,7 +39,7 @@ public class InternalGeoCentroid extends InternalAggregation implements GeoCentr protected final long count; public static long encodeLatLon(double lat, double lon) { - return (GeoEncodingUtils.encodeLatitude(lat) << 32) + GeoEncodingUtils.encodeLongitude(lon); + return (Integer.toUnsignedLong(GeoEncodingUtils.encodeLatitude(lat)) << 32) | Integer.toUnsignedLong(GeoEncodingUtils.encodeLongitude(lon)); } public static double decodeLatitude(long encodedLatLon) { @@ -47,7 +47,7 @@ public static double decodeLatitude(long encodedLatLon) { } public static double decodeLongitude(long encodedLatLon) { - return GeoEncodingUtils.decodeLatitude((int) encodedLatLon); + return GeoEncodingUtils.decodeLongitude((int) (encodedLatLon & 0xFFFFFFFFL)); } public InternalGeoCentroid(String name, GeoPoint centroid, long count, List diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/support/MissingValues.java b/core/src/main/java/org/elasticsearch/search/aggregations/support/MissingValues.java index fdd3701a13098..d5ff2cde97f87 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/support/MissingValues.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/support/MissingValues.java @@ -350,7 +350,7 @@ public int docValueCount() { } @Override - public GeoPoint nextValue() { + public GeoPoint nextValue() throws IOException { if (count > 0) { return values.nextValue(); } else { From 10868b021ce3241005688977bba540e422d887d2 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Wed, 12 Apr 2017 11:46:15 +0200 Subject: [PATCH 71/87] More fixes. --- core/src/main/java/org/elasticsearch/common/geo/GeoUtils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/org/elasticsearch/common/geo/GeoUtils.java b/core/src/main/java/org/elasticsearch/common/geo/GeoUtils.java index b1448796821a4..a586414631805 100644 --- a/core/src/main/java/org/elasticsearch/common/geo/GeoUtils.java +++ b/core/src/main/java/org/elasticsearch/common/geo/GeoUtils.java @@ -533,8 +533,8 @@ public boolean advanceExact(int target) throws IOException { resize(geoPointValues.docValueCount() * fromPoints.length); int v = 0; for (int i = 0; i < geoPointValues.docValueCount(); ++i) { + final GeoPoint point = geoPointValues.nextValue(); for (GeoPoint from : fromPoints) { - final GeoPoint point = geoPointValues.nextValue(); values[v] = distance.calculate(from.lat(), from.lon(), point.lat(), point.lon(), unit); v++; } From 531c8d3c91d2ede735c6760b103def7e4be0c121 Mon Sep 17 00:00:00 2001 From: Colin Goodheart-Smithe Date: Wed, 12 Apr 2017 10:17:51 +0100 Subject: [PATCH 72/87] Fixes dates in ScriptDocValuesLongsTests --- .../index/fielddata/ScriptDocValues.java | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java b/core/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java index d8e33da6b99a3..770c7ecdaaa0b 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java @@ -162,7 +162,7 @@ public static final class Longs extends ScriptDocValues { private long[] values = new long[0]; private int count; private Dates dates; - private int docId; + private int docId = -1; public Longs(SortedNumericDocValues in) { this.in = in; @@ -210,7 +210,11 @@ public ReadableDateTime getDate() throws IOException { deprecationLogger.deprecated("getDate on numeric fields is deprecated. Use a date field to get dates."); if (dates == null) { dates = new Dates(in); - dates.setNextDocId(docId); + if (docId >= 0) { + dates.setNextDocId(docId); + } else { + dates.refreshArray(); + } } return dates.getValue(); } @@ -220,7 +224,11 @@ public List getDates() throws IOException { deprecationLogger.deprecated("getDates on numeric fields is deprecated. Use a date field to get dates."); if (dates == null) { dates = new Dates(in); - dates.setNextDocId(docId); + if (docId >= 0) { + dates.setNextDocId(docId); + } else { + dates.refreshArray(); + } } return dates; } From 5731ebdb19fef01ada841c00f914adb3190dec4e Mon Sep 17 00:00:00 2001 From: Colin Goodheart-Smithe Date: Wed, 12 Apr 2017 10:31:19 +0000 Subject: [PATCH 73/87] Fixes JavaDocs error --- .../java/org/elasticsearch/index/fielddata/FieldData.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/FieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/FieldData.java index b39e6cb049eca..76ac003f99a03 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/FieldData.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/FieldData.java @@ -203,7 +203,7 @@ public int length() { * Given a {@link SortedNumericDoubleValues}, return a * {@link SortedNumericDocValues} instance that will translate double values * to sortable long bits using - * {@link NumericUtils#doubleToSortableLong(double)}. + * {@link org.apache.lucene.util.NumericUtils#doubleToSortableLong(double)}. */ public static SortedNumericDocValues toSortableLongBits(SortedNumericDoubleValues values) { final NumericDoubleValues singleton = unwrapSingleton(values); @@ -227,7 +227,7 @@ public static SortedNumericDocValues toSortableLongBits(SortedNumericDoubleValue /** * Given a {@link SortedNumericDocValues}, return a {@link SortedNumericDoubleValues} * instance that will translate long values to doubles using - * {@link NumericUtils#sortableLongToDouble(long)}. + * {@link org.apache.lucene.util.NumericUtils#sortableLongToDouble(long)}. */ public static SortedNumericDoubleValues sortableLongBitsToDoubles(SortedNumericDocValues values) { final NumericDocValues singleton = DocValues.unwrapSingleton(values); From 6a13ee3e7846339521e77584c7bf33145110916a Mon Sep 17 00:00:00 2001 From: Colin Goodheart-Smithe Date: Wed, 12 Apr 2017 12:10:00 +0000 Subject: [PATCH 74/87] More test fixes --- .../index/fielddata/ScriptDocValues.java | 31 +++++-------------- .../query/SimpleQueryStringBuilderTests.java | 5 +-- 2 files changed, 11 insertions(+), 25 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java b/core/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java index 770c7ecdaaa0b..1c6ae3a0e3306 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java @@ -23,7 +23,6 @@ import org.apache.lucene.index.SortedNumericDocValues; import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.BytesRefBuilder; import org.elasticsearch.common.geo.GeoHashUtils; import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.common.geo.GeoUtils; @@ -88,7 +87,7 @@ public final void sort(Comparator c) { public static final class Strings extends ScriptDocValues { private final SortedBinaryDocValues in; - private BytesRefBuilder[] values = new BytesRefBuilder[0]; + private BytesRef[] values = new BytesRef[0]; private int count; public Strings(SortedBinaryDocValues in) { @@ -100,7 +99,7 @@ public void setNextDocId(int docId) throws IOException { if (in.advanceExact(docId)) { resize(in.docValueCount()); for (int i = 0; i < count; i++) { - values[i].copyBytes(in.nextValue()); + values[i] = in.nextValue(); } } else { resize(0); @@ -113,13 +112,7 @@ public void setNextDocId(int docId) throws IOException { */ protected void resize(int newSize) { count = newSize; - if (newSize > values.length) { - final int oldLength = values.length; - values = ArrayUtil.grow(values, count); - for (int i = oldLength; i < values.length; ++i) { - values[i] = new BytesRefBuilder(); - } - } + values = ArrayUtil.grow(values, count); } public SortedBinaryDocValues getInternalValues() { @@ -128,7 +121,7 @@ public SortedBinaryDocValues getInternalValues() { public BytesRef getBytesValue() { if (size() > 0) { - return values[0].get(); + return values[0]; } else { return null; } @@ -145,7 +138,7 @@ public String getValue() { @Override public String get(int index) { - return values[index].get().utf8ToString(); + return values[index].utf8ToString(); } @Override @@ -171,8 +164,8 @@ public Longs(SortedNumericDocValues in) { @Override public void setNextDocId(int docId) throws IOException { + this.docId = docId; if (in.advanceExact(docId)) { - this.docId = docId; resize(in.docValueCount()); for (int i = 0; i < count; i++) { values[i] = in.nextValue(); @@ -210,11 +203,7 @@ public ReadableDateTime getDate() throws IOException { deprecationLogger.deprecated("getDate on numeric fields is deprecated. Use a date field to get dates."); if (dates == null) { dates = new Dates(in); - if (docId >= 0) { - dates.setNextDocId(docId); - } else { - dates.refreshArray(); - } + dates.setNextDocId(docId); } return dates.getValue(); } @@ -224,11 +213,7 @@ public List getDates() throws IOException { deprecationLogger.deprecated("getDates on numeric fields is deprecated. Use a date field to get dates."); if (dates == null) { dates = new Dates(in); - if (docId >= 0) { - dates.setNextDocId(docId); - } else { - dates.refreshArray(); - } + dates.setNextDocId(docId); } return dates; } diff --git a/core/src/test/java/org/elasticsearch/index/query/SimpleQueryStringBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/SimpleQueryStringBuilderTests.java index aa515efd0020f..e08221a168c2a 100644 --- a/core/src/test/java/org/elasticsearch/index/query/SimpleQueryStringBuilderTests.java +++ b/core/src/test/java/org/elasticsearch/index/query/SimpleQueryStringBuilderTests.java @@ -47,7 +47,6 @@ import static org.hamcrest.Matchers.anyOf; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; @@ -338,6 +337,7 @@ public void testFromJson() throws IOException { assertEquals(json, ".quote", parsed.quoteFieldSuffix()); } + @AwaitsFix(bugUrl = "Waiting on fix for minimumShouldMatch https://github.com/elastic/elasticsearch/issues/23966") public void testMinimumShouldMatch() throws IOException { QueryShardContext shardContext = createShardContext(); int numberOfTerms = randomIntBetween(1, 4); @@ -364,7 +364,8 @@ public void testMinimumShouldMatch() throws IOException { assertThat(query, instanceOf(BooleanQuery.class)); BooleanQuery boolQuery = (BooleanQuery) query; int expectedMinimumShouldMatch = numberOfTerms * percent / 100; - if (numberOfTerms == 1 || simpleQueryStringBuilder.defaultOperator().equals(Operator.AND)) { + if (numberOfTerms == 1 + || simpleQueryStringBuilder.defaultOperator().equals(Operator.AND)) { expectedMinimumShouldMatch = 0; } assertEquals(expectedMinimumShouldMatch, boolQuery.getMinimumNumberShouldMatch()); From 885d2115f8f2f104b39146e70f0da4662fddb315 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Wed, 12 Apr 2017 15:03:38 +0200 Subject: [PATCH 75/87] More fixes. --- .../index/fielddata/ScriptDocValues.java | 17 ++++++++---- .../GlobalOrdinalsStringTermsAggregator.java | 27 +++++++++---------- .../support/values/ScriptBytesValues.java | 3 --- 3 files changed, 24 insertions(+), 23 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java b/core/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java index 1c6ae3a0e3306..339e70c50b1c8 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java @@ -23,6 +23,7 @@ import org.apache.lucene.index.SortedNumericDocValues; import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.BytesRefBuilder; import org.elasticsearch.common.geo.GeoHashUtils; import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.common.geo.GeoUtils; @@ -87,7 +88,7 @@ public final void sort(Comparator c) { public static final class Strings extends ScriptDocValues { private final SortedBinaryDocValues in; - private BytesRef[] values = new BytesRef[0]; + private BytesRefBuilder[] values = new BytesRefBuilder[0]; private int count; public Strings(SortedBinaryDocValues in) { @@ -99,7 +100,7 @@ public void setNextDocId(int docId) throws IOException { if (in.advanceExact(docId)) { resize(in.docValueCount()); for (int i = 0; i < count; i++) { - values[i] = in.nextValue(); + values[i].copyBytes(in.nextValue()); } } else { resize(0); @@ -112,7 +113,13 @@ public void setNextDocId(int docId) throws IOException { */ protected void resize(int newSize) { count = newSize; - values = ArrayUtil.grow(values, count); + if (newSize > values.length) { + final int oldLength = values.length; + values = ArrayUtil.grow(values, count); + for (int i = oldLength; i < values.length; ++i) { + values[i] = new BytesRefBuilder(); + } + } } public SortedBinaryDocValues getInternalValues() { @@ -121,7 +128,7 @@ public SortedBinaryDocValues getInternalValues() { public BytesRef getBytesValue() { if (size() > 0) { - return values[0]; + return values[0].get(); } else { return null; } @@ -138,7 +145,7 @@ public String getValue() { @Override public String get(int index) { - return values[index].utf8ToString(); + return values[index].get().utf8ToString(); } @Override diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/GlobalOrdinalsStringTermsAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/GlobalOrdinalsStringTermsAggregator.java index cb00bdf93f222..5ef96d19257cc 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/GlobalOrdinalsStringTermsAggregator.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/GlobalOrdinalsStringTermsAggregator.java @@ -435,10 +435,6 @@ private static final class FilteredOrdinals extends AbstractSortedSetDocValues { private final SortedSetDocValues inner; private final LongBitSet accepted; - private int cardinality; - private long[] ords = new long[0]; - private int ordsCursor = 0; - private FilteredOrdinals(SortedSetDocValues inner, LongBitSet accepted) { this.inner = inner; this.accepted = accepted; @@ -456,26 +452,27 @@ public BytesRef lookupOrd(long ord) throws IOException { @Override public long nextOrd() throws IOException { - return ords[ordsCursor++]; + for (long ord = inner.nextOrd(); ord != NO_MORE_ORDS; ord = inner.nextOrd()) { + if (accepted.get(ord)) { + return ord; + } + } + return NO_MORE_ORDS; } @Override public boolean advanceExact(int target) throws IOException { if (inner.advanceExact(target)) { - ordsCursor = 0; - - cardinality = 0; - for (long ord = inner.nextOrd(); ord != SortedSetDocValues.NO_MORE_ORDS; - ord = inner.nextOrd()) { + for (long ord = inner.nextOrd(); ord != NO_MORE_ORDS; ord = inner.nextOrd()) { if (accepted.get(ord)) { - ords = ArrayUtil.grow(ords, cardinality + 1); - ords[cardinality++] = ord; + // reset the iterator + boolean advanced = inner.advanceExact(target); + assert advanced; + return true; } } - return true; - } else { - return false; } + return false; } } } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/support/values/ScriptBytesValues.java b/core/src/main/java/org/elasticsearch/search/aggregations/support/values/ScriptBytesValues.java index 9e0b3b9ef6d3b..78685ed0e82ca 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/support/values/ScriptBytesValues.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/support/values/ScriptBytesValues.java @@ -65,9 +65,6 @@ public boolean advanceExact(int doc) throws IOException { } } else if (value instanceof Collection) { final Collection coll = (Collection) value; - if (coll.isEmpty()) { - return false; - } count = coll.size(); if (count == 0) { return false; From 8b06cf7f88726eb3e987dfc68441ed3a94caa1c2 Mon Sep 17 00:00:00 2001 From: Colin Goodheart-Smithe Date: Wed, 12 Apr 2017 13:37:37 +0000 Subject: [PATCH 76/87] Fixes Expression unit tests --- .../org/elasticsearch/search/MultiValueMode.java | 2 +- .../search/aggregations/support/ValuesSource.java | 2 +- .../script/expression/EmptyMemberValueSource.java | 12 ++++++------ 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/search/MultiValueMode.java b/core/src/main/java/org/elasticsearch/search/MultiValueMode.java index acccb6d71d684..2d6fd8a2b603a 100644 --- a/core/src/main/java/org/elasticsearch/search/MultiValueMode.java +++ b/core/src/main/java/org/elasticsearch/search/MultiValueMode.java @@ -213,7 +213,7 @@ protected double pick(SortedNumericDoubleValues values) throws IOException { values.nextValue(); } if (count % 2 == 0) { - return Math.round((values.nextValue() + values.nextValue()) / 2); + return (values.nextValue() + values.nextValue()) / 2; } else { return values.nextValue(); } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSource.java b/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSource.java index 7fcc7595ea62c..e6ed81c2d83af 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSource.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSource.java @@ -356,7 +356,7 @@ public boolean advanceExact(int target) throws IOException { script.setDocument(target); for (int i = 0; i < docValueCount(); ++i) { script.setNextAggregationValue(doubleValues.nextValue()); - values[i] = script.runAsLong(); + values[i] = script.runAsDouble(); } sort(); return true; diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/EmptyMemberValueSource.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/EmptyMemberValueSource.java index 0c912c6b0d83c..aa12560e1fce0 100644 --- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/EmptyMemberValueSource.java +++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/EmptyMemberValueSource.java @@ -19,10 +19,6 @@ package org.elasticsearch.script.expression; -import java.io.IOException; -import java.util.Map; -import java.util.Objects; - import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.queries.function.FunctionValues; import org.apache.lucene.queries.function.ValueSource; @@ -31,6 +27,10 @@ import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; +import java.io.IOException; +import java.util.Map; +import java.util.Objects; + /** * ValueSource to return non-zero if a field is missing. *

@@ -52,9 +52,9 @@ public FunctionValues getValues(Map context, LeafReaderContext leaf) throws IOEx @Override public double doubleVal(int doc) throws IOException { if (values.advanceExact(doc)) { - return 1; - } else { return 0; + } else { + return 1; } } }; From 72af5915985603043ac2dcc1891b37cbe82b4542 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Wed, 12 Apr 2017 16:11:44 +0200 Subject: [PATCH 77/87] Add AwaitsFix. --- .../java/org/elasticsearch/search/query/SimpleQueryStringIT.java | 1 + 1 file changed, 1 insertion(+) diff --git a/core/src/test/java/org/elasticsearch/search/query/SimpleQueryStringIT.java b/core/src/test/java/org/elasticsearch/search/query/SimpleQueryStringIT.java index f22ec392b9953..58c0bf82e98d1 100644 --- a/core/src/test/java/org/elasticsearch/search/query/SimpleQueryStringIT.java +++ b/core/src/test/java/org/elasticsearch/search/query/SimpleQueryStringIT.java @@ -120,6 +120,7 @@ public void testSimpleQueryString() throws ExecutionException, InterruptedExcept assertSearchHits(searchResponse, "5", "6"); } + @AwaitsFix(bugUrl="https://github.com/elastic/elasticsearch/issues/23966") public void testSimpleQueryStringMinimumShouldMatch() throws Exception { createIndex("test"); ensureGreen("test"); From 874800ceba384bc208da9d70a2a722528b570f1e Mon Sep 17 00:00:00 2001 From: Colin Goodheart-Smithe Date: Wed, 12 Apr 2017 15:06:35 +0000 Subject: [PATCH 78/87] fixes integTest looking for coords --- .../resources/rest-api-spec/test/search_shards/10_basic.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search_shards/10_basic.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/search_shards/10_basic.yaml index d48d50887569c..42189883b1bab 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search_shards/10_basic.yaml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search_shards/10_basic.yaml @@ -76,6 +76,5 @@ - lte: { indices.test_index.filter.bool.should.1.term.field.boost: 1.0 } - gte: { indices.test_index.filter.bool.should.1.term.field.boost: 1.0 } - match: { indices.test_index.filter.bool.adjust_pure_negative: true} - - match: { indices.test_index.filter.bool.disable_coord: false} - lte: { indices.test_index.filter.bool.boost: 1.0 } - gte: { indices.test_index.filter.bool.boost: 1.0 } From c93fb974c8690400b2cd16123a24de1bfc1d64b5 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Thu, 13 Apr 2017 09:10:42 +0200 Subject: [PATCH 79/87] Fix matrix stats failures. --- .../aggregations/matrix/stats/MatrixStatsAggregator.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsAggregator.java b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsAggregator.java index df693fc25140e..5c193828c5536 100644 --- a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsAggregator.java +++ b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsAggregator.java @@ -106,6 +106,10 @@ private boolean includeDocument(int doc) throws IOException { final NumericDoubleValues doubleValues = values[i]; if (doubleValues.advanceExact(doc)) { final double value = doubleValues.doubleValue(); + if (value == Double.NEGATIVE_INFINITY) { + // TODO: Fix matrix stats to treat neg inf as any other value + return false; + } fieldVals[i] = value; } else { return false; From f15fe7c20c2870086822a2f45ecf46377b7a0b42 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Thu, 13 Apr 2017 10:17:19 +0200 Subject: [PATCH 80/87] Update the Lucene snapshot. --- buildSrc/version.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildSrc/version.properties b/buildSrc/version.properties index f0ddb1b271db5..ca263b0efa93e 100644 --- a/buildSrc/version.properties +++ b/buildSrc/version.properties @@ -1,6 +1,6 @@ # When updating elasticsearch, please update 'rest' version in core/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy elasticsearch = 6.0.0-alpha1 -lucene = 7.0.0-snapshot-05749d0 +lucene = 7.0.0-snapshot-89f6d17 # optional dependencies spatial4j = 0.6 From 1eb6d86da4cb3846c1361f2e21b8a24d99645498 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Thu, 13 Apr 2017 10:20:21 +0200 Subject: [PATCH 81/87] Fix some compile errors. --- .../fieldcomparator/BytesRefFieldComparatorSource.java | 2 +- .../index/fielddata/plain/ParentChildIndexFieldData.java | 2 +- .../search/fetch/subphase/InnerHitsContext.java | 8 ++------ 3 files changed, 4 insertions(+), 8 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/BytesRefFieldComparatorSource.java b/core/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/BytesRefFieldComparatorSource.java index 87cbb7d86c4e8..c433836417a0f 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/BytesRefFieldComparatorSource.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/BytesRefFieldComparatorSource.java @@ -163,7 +163,7 @@ static class ReplaceMissing extends AbstractSortedDocValues { } @Override - public int ordValue() { + public int ordValue() throws IOException { if (hasValue == false) { return substituteOrd; } diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/ParentChildIndexFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/ParentChildIndexFieldData.java index 381a7ae9435fe..74b180f2c1bae 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/ParentChildIndexFieldData.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/plain/ParentChildIndexFieldData.java @@ -270,7 +270,7 @@ public int getValueCount() { } @Override - public int ordValue() { + public int ordValue() throws IOException { return (int) globalOrds.get(segmentValues.ordValue()); } diff --git a/core/src/main/java/org/elasticsearch/search/fetch/subphase/InnerHitsContext.java b/core/src/main/java/org/elasticsearch/search/fetch/subphase/InnerHitsContext.java index cad147f28d1fd..aa4168bd0402b 100644 --- a/core/src/main/java/org/elasticsearch/search/fetch/subphase/InnerHitsContext.java +++ b/core/src/main/java/org/elasticsearch/search/fetch/subphase/InnerHitsContext.java @@ -130,13 +130,9 @@ public TopDocs topDocs(SearchContext context, FetchSubPhase.HitContext hitContex return new TopDocs(context.searcher().count(q), Lucene.EMPTY_SCORE_DOCS, 0); } else { int topN = Math.min(from() + size(), context.searcher().getIndexReader().maxDoc()); - TopDocsCollector topDocsCollector; + TopDocsCollector topDocsCollector; if (sort() != null) { - try { - topDocsCollector = TopFieldCollector.create(sort().sort, topN, true, trackScores(), trackScores()); - } catch (IOException e) { - throw ExceptionsHelper.convertToElastic(e); - } + topDocsCollector = TopFieldCollector.create(sort().sort, topN, true, trackScores(), trackScores()); } else { topDocsCollector = TopScoreDocCollector.create(topN); } From 08404e69888ae0334cee335e2851200d3ff8a4b0 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Thu, 13 Apr 2017 10:45:05 +0200 Subject: [PATCH 82/87] Finish Lucene upgrade. --- .../lucene-analyzers-common-7.0.0-snapshot-05749d0.jar.sha1 | 1 - .../lucene-analyzers-common-7.0.0-snapshot-89f6d17.jar.sha1 | 1 + .../lucene-backward-codecs-7.0.0-snapshot-05749d0.jar.sha1 | 1 - .../lucene-backward-codecs-7.0.0-snapshot-89f6d17.jar.sha1 | 1 + core/licenses/lucene-core-7.0.0-snapshot-05749d0.jar.sha1 | 1 - core/licenses/lucene-core-7.0.0-snapshot-89f6d17.jar.sha1 | 1 + core/licenses/lucene-grouping-7.0.0-snapshot-05749d0.jar.sha1 | 1 - core/licenses/lucene-grouping-7.0.0-snapshot-89f6d17.jar.sha1 | 1 + .../lucene-highlighter-7.0.0-snapshot-05749d0.jar.sha1 | 1 - .../lucene-highlighter-7.0.0-snapshot-89f6d17.jar.sha1 | 1 + core/licenses/lucene-join-7.0.0-snapshot-05749d0.jar.sha1 | 1 - core/licenses/lucene-join-7.0.0-snapshot-89f6d17.jar.sha1 | 1 + core/licenses/lucene-memory-7.0.0-snapshot-05749d0.jar.sha1 | 1 - core/licenses/lucene-memory-7.0.0-snapshot-89f6d17.jar.sha1 | 1 + core/licenses/lucene-misc-7.0.0-snapshot-05749d0.jar.sha1 | 1 - core/licenses/lucene-misc-7.0.0-snapshot-89f6d17.jar.sha1 | 1 + core/licenses/lucene-queries-7.0.0-snapshot-05749d0.jar.sha1 | 1 - core/licenses/lucene-queries-7.0.0-snapshot-89f6d17.jar.sha1 | 1 + .../lucene-queryparser-7.0.0-snapshot-05749d0.jar.sha1 | 1 - .../lucene-queryparser-7.0.0-snapshot-89f6d17.jar.sha1 | 1 + core/licenses/lucene-sandbox-7.0.0-snapshot-05749d0.jar.sha1 | 1 - core/licenses/lucene-sandbox-7.0.0-snapshot-89f6d17.jar.sha1 | 1 + core/licenses/lucene-spatial-7.0.0-snapshot-05749d0.jar.sha1 | 1 - core/licenses/lucene-spatial-7.0.0-snapshot-89f6d17.jar.sha1 | 1 + .../lucene-spatial-extras-7.0.0-snapshot-05749d0.jar.sha1 | 1 - .../lucene-spatial-extras-7.0.0-snapshot-89f6d17.jar.sha1 | 1 + .../licenses/lucene-spatial3d-7.0.0-snapshot-05749d0.jar.sha1 | 1 - .../licenses/lucene-spatial3d-7.0.0-snapshot-89f6d17.jar.sha1 | 1 + core/licenses/lucene-suggest-7.0.0-snapshot-05749d0.jar.sha1 | 1 - core/licenses/lucene-suggest-7.0.0-snapshot-89f6d17.jar.sha1 | 1 + .../resources/org/elasticsearch/bootstrap/security.policy | 4 ++-- .../org/elasticsearch/bootstrap/test-framework.policy | 2 +- .../lucene-expressions-7.0.0-snapshot-05749d0.jar.sha1 | 1 - .../lucene-expressions-7.0.0-snapshot-89f6d17.jar.sha1 | 1 + .../lucene-analyzers-icu-7.0.0-snapshot-05749d0.jar.sha1 | 1 - .../lucene-analyzers-icu-7.0.0-snapshot-89f6d17.jar.sha1 | 1 + .../lucene-analyzers-kuromoji-7.0.0-snapshot-05749d0.jar.sha1 | 1 - .../lucene-analyzers-kuromoji-7.0.0-snapshot-89f6d17.jar.sha1 | 1 + .../lucene-analyzers-phonetic-7.0.0-snapshot-05749d0.jar.sha1 | 1 - .../lucene-analyzers-phonetic-7.0.0-snapshot-89f6d17.jar.sha1 | 1 + .../lucene-analyzers-smartcn-7.0.0-snapshot-05749d0.jar.sha1 | 1 - .../lucene-analyzers-smartcn-7.0.0-snapshot-89f6d17.jar.sha1 | 1 + .../lucene-analyzers-stempel-7.0.0-snapshot-05749d0.jar.sha1 | 1 - .../lucene-analyzers-stempel-7.0.0-snapshot-89f6d17.jar.sha1 | 1 + ...ucene-analyzers-morfologik-7.0.0-snapshot-05749d0.jar.sha1 | 1 - ...ucene-analyzers-morfologik-7.0.0-snapshot-89f6d17.jar.sha1 | 1 + 46 files changed, 25 insertions(+), 25 deletions(-) delete mode 100644 core/licenses/lucene-analyzers-common-7.0.0-snapshot-05749d0.jar.sha1 create mode 100644 core/licenses/lucene-analyzers-common-7.0.0-snapshot-89f6d17.jar.sha1 delete mode 100644 core/licenses/lucene-backward-codecs-7.0.0-snapshot-05749d0.jar.sha1 create mode 100644 core/licenses/lucene-backward-codecs-7.0.0-snapshot-89f6d17.jar.sha1 delete mode 100644 core/licenses/lucene-core-7.0.0-snapshot-05749d0.jar.sha1 create mode 100644 core/licenses/lucene-core-7.0.0-snapshot-89f6d17.jar.sha1 delete mode 100644 core/licenses/lucene-grouping-7.0.0-snapshot-05749d0.jar.sha1 create mode 100644 core/licenses/lucene-grouping-7.0.0-snapshot-89f6d17.jar.sha1 delete mode 100644 core/licenses/lucene-highlighter-7.0.0-snapshot-05749d0.jar.sha1 create mode 100644 core/licenses/lucene-highlighter-7.0.0-snapshot-89f6d17.jar.sha1 delete mode 100644 core/licenses/lucene-join-7.0.0-snapshot-05749d0.jar.sha1 create mode 100644 core/licenses/lucene-join-7.0.0-snapshot-89f6d17.jar.sha1 delete mode 100644 core/licenses/lucene-memory-7.0.0-snapshot-05749d0.jar.sha1 create mode 100644 core/licenses/lucene-memory-7.0.0-snapshot-89f6d17.jar.sha1 delete mode 100644 core/licenses/lucene-misc-7.0.0-snapshot-05749d0.jar.sha1 create mode 100644 core/licenses/lucene-misc-7.0.0-snapshot-89f6d17.jar.sha1 delete mode 100644 core/licenses/lucene-queries-7.0.0-snapshot-05749d0.jar.sha1 create mode 100644 core/licenses/lucene-queries-7.0.0-snapshot-89f6d17.jar.sha1 delete mode 100644 core/licenses/lucene-queryparser-7.0.0-snapshot-05749d0.jar.sha1 create mode 100644 core/licenses/lucene-queryparser-7.0.0-snapshot-89f6d17.jar.sha1 delete mode 100644 core/licenses/lucene-sandbox-7.0.0-snapshot-05749d0.jar.sha1 create mode 100644 core/licenses/lucene-sandbox-7.0.0-snapshot-89f6d17.jar.sha1 delete mode 100644 core/licenses/lucene-spatial-7.0.0-snapshot-05749d0.jar.sha1 create mode 100644 core/licenses/lucene-spatial-7.0.0-snapshot-89f6d17.jar.sha1 delete mode 100644 core/licenses/lucene-spatial-extras-7.0.0-snapshot-05749d0.jar.sha1 create mode 100644 core/licenses/lucene-spatial-extras-7.0.0-snapshot-89f6d17.jar.sha1 delete mode 100644 core/licenses/lucene-spatial3d-7.0.0-snapshot-05749d0.jar.sha1 create mode 100644 core/licenses/lucene-spatial3d-7.0.0-snapshot-89f6d17.jar.sha1 delete mode 100644 core/licenses/lucene-suggest-7.0.0-snapshot-05749d0.jar.sha1 create mode 100644 core/licenses/lucene-suggest-7.0.0-snapshot-89f6d17.jar.sha1 delete mode 100644 modules/lang-expression/licenses/lucene-expressions-7.0.0-snapshot-05749d0.jar.sha1 create mode 100644 modules/lang-expression/licenses/lucene-expressions-7.0.0-snapshot-89f6d17.jar.sha1 delete mode 100644 plugins/analysis-icu/licenses/lucene-analyzers-icu-7.0.0-snapshot-05749d0.jar.sha1 create mode 100644 plugins/analysis-icu/licenses/lucene-analyzers-icu-7.0.0-snapshot-89f6d17.jar.sha1 delete mode 100644 plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-7.0.0-snapshot-05749d0.jar.sha1 create mode 100644 plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-7.0.0-snapshot-89f6d17.jar.sha1 delete mode 100644 plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-7.0.0-snapshot-05749d0.jar.sha1 create mode 100644 plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-7.0.0-snapshot-89f6d17.jar.sha1 delete mode 100644 plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-7.0.0-snapshot-05749d0.jar.sha1 create mode 100644 plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-7.0.0-snapshot-89f6d17.jar.sha1 delete mode 100644 plugins/analysis-stempel/licenses/lucene-analyzers-stempel-7.0.0-snapshot-05749d0.jar.sha1 create mode 100644 plugins/analysis-stempel/licenses/lucene-analyzers-stempel-7.0.0-snapshot-89f6d17.jar.sha1 delete mode 100644 plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-7.0.0-snapshot-05749d0.jar.sha1 create mode 100644 plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-7.0.0-snapshot-89f6d17.jar.sha1 diff --git a/core/licenses/lucene-analyzers-common-7.0.0-snapshot-05749d0.jar.sha1 b/core/licenses/lucene-analyzers-common-7.0.0-snapshot-05749d0.jar.sha1 deleted file mode 100644 index 37783c4b629fb..0000000000000 --- a/core/licenses/lucene-analyzers-common-7.0.0-snapshot-05749d0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -364580e75893da429db6314cbbe3810afc30dfdc \ No newline at end of file diff --git a/core/licenses/lucene-analyzers-common-7.0.0-snapshot-89f6d17.jar.sha1 b/core/licenses/lucene-analyzers-common-7.0.0-snapshot-89f6d17.jar.sha1 new file mode 100644 index 0000000000000..93afa8a73f448 --- /dev/null +++ b/core/licenses/lucene-analyzers-common-7.0.0-snapshot-89f6d17.jar.sha1 @@ -0,0 +1 @@ +e69234c2e898d86a53edbe8d22e33bebc45286cd \ No newline at end of file diff --git a/core/licenses/lucene-backward-codecs-7.0.0-snapshot-05749d0.jar.sha1 b/core/licenses/lucene-backward-codecs-7.0.0-snapshot-05749d0.jar.sha1 deleted file mode 100644 index a5faad824dc2d..0000000000000 --- a/core/licenses/lucene-backward-codecs-7.0.0-snapshot-05749d0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -af62e6aad7490205e350d986a53b495c384df5ac \ No newline at end of file diff --git a/core/licenses/lucene-backward-codecs-7.0.0-snapshot-89f6d17.jar.sha1 b/core/licenses/lucene-backward-codecs-7.0.0-snapshot-89f6d17.jar.sha1 new file mode 100644 index 0000000000000..707ed41267f52 --- /dev/null +++ b/core/licenses/lucene-backward-codecs-7.0.0-snapshot-89f6d17.jar.sha1 @@ -0,0 +1 @@ +48172a8e1fe6562f55ab671d42af53652794d5df \ No newline at end of file diff --git a/core/licenses/lucene-core-7.0.0-snapshot-05749d0.jar.sha1 b/core/licenses/lucene-core-7.0.0-snapshot-05749d0.jar.sha1 deleted file mode 100644 index 08054972a2a1c..0000000000000 --- a/core/licenses/lucene-core-7.0.0-snapshot-05749d0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -31f5d32778e1dbcc909f4245cdfd691d2f16f7d5 \ No newline at end of file diff --git a/core/licenses/lucene-core-7.0.0-snapshot-89f6d17.jar.sha1 b/core/licenses/lucene-core-7.0.0-snapshot-89f6d17.jar.sha1 new file mode 100644 index 0000000000000..ef6a939a6688d --- /dev/null +++ b/core/licenses/lucene-core-7.0.0-snapshot-89f6d17.jar.sha1 @@ -0,0 +1 @@ +3dab251d4c7ab4ff5095e5f1d1e127ec2cf3c07d \ No newline at end of file diff --git a/core/licenses/lucene-grouping-7.0.0-snapshot-05749d0.jar.sha1 b/core/licenses/lucene-grouping-7.0.0-snapshot-05749d0.jar.sha1 deleted file mode 100644 index e4d38ee0e9995..0000000000000 --- a/core/licenses/lucene-grouping-7.0.0-snapshot-05749d0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -76721f38a42d18694be50d765dc34e0aa8fd0f7d \ No newline at end of file diff --git a/core/licenses/lucene-grouping-7.0.0-snapshot-89f6d17.jar.sha1 b/core/licenses/lucene-grouping-7.0.0-snapshot-89f6d17.jar.sha1 new file mode 100644 index 0000000000000..dfa06b60b4e84 --- /dev/null +++ b/core/licenses/lucene-grouping-7.0.0-snapshot-89f6d17.jar.sha1 @@ -0,0 +1 @@ +c01ae8a23b733d75d058a76bd85fcb49b9fd06fd \ No newline at end of file diff --git a/core/licenses/lucene-highlighter-7.0.0-snapshot-05749d0.jar.sha1 b/core/licenses/lucene-highlighter-7.0.0-snapshot-05749d0.jar.sha1 deleted file mode 100644 index 626d86e955f29..0000000000000 --- a/core/licenses/lucene-highlighter-7.0.0-snapshot-05749d0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -5bb097fe7d91ed39544b8c2042e7f9de0a4cb896 \ No newline at end of file diff --git a/core/licenses/lucene-highlighter-7.0.0-snapshot-89f6d17.jar.sha1 b/core/licenses/lucene-highlighter-7.0.0-snapshot-89f6d17.jar.sha1 new file mode 100644 index 0000000000000..51c8a47474f93 --- /dev/null +++ b/core/licenses/lucene-highlighter-7.0.0-snapshot-89f6d17.jar.sha1 @@ -0,0 +1 @@ +c53df048b97946fe66035505306b5651b702adb1 \ No newline at end of file diff --git a/core/licenses/lucene-join-7.0.0-snapshot-05749d0.jar.sha1 b/core/licenses/lucene-join-7.0.0-snapshot-05749d0.jar.sha1 deleted file mode 100644 index 9505d2beb9f75..0000000000000 --- a/core/licenses/lucene-join-7.0.0-snapshot-05749d0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -95ddeed8524ea4feaca455b962ca1883735d241f \ No newline at end of file diff --git a/core/licenses/lucene-join-7.0.0-snapshot-89f6d17.jar.sha1 b/core/licenses/lucene-join-7.0.0-snapshot-89f6d17.jar.sha1 new file mode 100644 index 0000000000000..ad7e123f6dfdb --- /dev/null +++ b/core/licenses/lucene-join-7.0.0-snapshot-89f6d17.jar.sha1 @@ -0,0 +1 @@ +1ecb349ba29abab75359e5125ac8a94fc81441d5 \ No newline at end of file diff --git a/core/licenses/lucene-memory-7.0.0-snapshot-05749d0.jar.sha1 b/core/licenses/lucene-memory-7.0.0-snapshot-05749d0.jar.sha1 deleted file mode 100644 index 8e5290de79fdc..0000000000000 --- a/core/licenses/lucene-memory-7.0.0-snapshot-05749d0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -3057c32455677c4656f681757a696b8eaac0f36f \ No newline at end of file diff --git a/core/licenses/lucene-memory-7.0.0-snapshot-89f6d17.jar.sha1 b/core/licenses/lucene-memory-7.0.0-snapshot-89f6d17.jar.sha1 new file mode 100644 index 0000000000000..6c9fc8be424bd --- /dev/null +++ b/core/licenses/lucene-memory-7.0.0-snapshot-89f6d17.jar.sha1 @@ -0,0 +1 @@ +e5f53b38652b1284ff254fba39e624ec117aef7d \ No newline at end of file diff --git a/core/licenses/lucene-misc-7.0.0-snapshot-05749d0.jar.sha1 b/core/licenses/lucene-misc-7.0.0-snapshot-05749d0.jar.sha1 deleted file mode 100644 index 190bf1549ad4a..0000000000000 --- a/core/licenses/lucene-misc-7.0.0-snapshot-05749d0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -0ef5a3ba35c6a3c3511407918d9d499ce55ea2ff \ No newline at end of file diff --git a/core/licenses/lucene-misc-7.0.0-snapshot-89f6d17.jar.sha1 b/core/licenses/lucene-misc-7.0.0-snapshot-89f6d17.jar.sha1 new file mode 100644 index 0000000000000..d69e68cdde630 --- /dev/null +++ b/core/licenses/lucene-misc-7.0.0-snapshot-89f6d17.jar.sha1 @@ -0,0 +1 @@ +2f340ed3f46d6b4c89fa31975b675c19028c15eb \ No newline at end of file diff --git a/core/licenses/lucene-queries-7.0.0-snapshot-05749d0.jar.sha1 b/core/licenses/lucene-queries-7.0.0-snapshot-05749d0.jar.sha1 deleted file mode 100644 index 56010255eac1a..0000000000000 --- a/core/licenses/lucene-queries-7.0.0-snapshot-05749d0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -7f8b9c2d214cc60fbc0e1bf8d6a140d514ee8100 \ No newline at end of file diff --git a/core/licenses/lucene-queries-7.0.0-snapshot-89f6d17.jar.sha1 b/core/licenses/lucene-queries-7.0.0-snapshot-89f6d17.jar.sha1 new file mode 100644 index 0000000000000..1d2f98e6c8cb8 --- /dev/null +++ b/core/licenses/lucene-queries-7.0.0-snapshot-89f6d17.jar.sha1 @@ -0,0 +1 @@ +a13862fb62cc1e516d16d6b6bb3cdb906c4925f6 \ No newline at end of file diff --git a/core/licenses/lucene-queryparser-7.0.0-snapshot-05749d0.jar.sha1 b/core/licenses/lucene-queryparser-7.0.0-snapshot-05749d0.jar.sha1 deleted file mode 100644 index e630517801ef3..0000000000000 --- a/core/licenses/lucene-queryparser-7.0.0-snapshot-05749d0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -f288aa637a02b074f9fcbff25debe37549a7dfd3 \ No newline at end of file diff --git a/core/licenses/lucene-queryparser-7.0.0-snapshot-89f6d17.jar.sha1 b/core/licenses/lucene-queryparser-7.0.0-snapshot-89f6d17.jar.sha1 new file mode 100644 index 0000000000000..29492f1ed6bc3 --- /dev/null +++ b/core/licenses/lucene-queryparser-7.0.0-snapshot-89f6d17.jar.sha1 @@ -0,0 +1 @@ +4e014f72a588453bae7dd1a555d741cf3bf39032 \ No newline at end of file diff --git a/core/licenses/lucene-sandbox-7.0.0-snapshot-05749d0.jar.sha1 b/core/licenses/lucene-sandbox-7.0.0-snapshot-05749d0.jar.sha1 deleted file mode 100644 index c96e2aa6d7f3a..0000000000000 --- a/core/licenses/lucene-sandbox-7.0.0-snapshot-05749d0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -1da8f0a2f4ec16b20300401357396c832ef9b501 \ No newline at end of file diff --git a/core/licenses/lucene-sandbox-7.0.0-snapshot-89f6d17.jar.sha1 b/core/licenses/lucene-sandbox-7.0.0-snapshot-89f6d17.jar.sha1 new file mode 100644 index 0000000000000..80794b3e272e2 --- /dev/null +++ b/core/licenses/lucene-sandbox-7.0.0-snapshot-89f6d17.jar.sha1 @@ -0,0 +1 @@ +5e87d61c604d6b1c0ee5c38f09441d1b8b9c8c2b \ No newline at end of file diff --git a/core/licenses/lucene-spatial-7.0.0-snapshot-05749d0.jar.sha1 b/core/licenses/lucene-spatial-7.0.0-snapshot-05749d0.jar.sha1 deleted file mode 100644 index 355ecc2b23116..0000000000000 --- a/core/licenses/lucene-spatial-7.0.0-snapshot-05749d0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -4a9780f5a2446d8d4f92919ce1e8e73d73fa2698 \ No newline at end of file diff --git a/core/licenses/lucene-spatial-7.0.0-snapshot-89f6d17.jar.sha1 b/core/licenses/lucene-spatial-7.0.0-snapshot-89f6d17.jar.sha1 new file mode 100644 index 0000000000000..859a89f5a0232 --- /dev/null +++ b/core/licenses/lucene-spatial-7.0.0-snapshot-89f6d17.jar.sha1 @@ -0,0 +1 @@ +be14aa163b339403d8ec904493c1be5dfa9baeaf \ No newline at end of file diff --git a/core/licenses/lucene-spatial-extras-7.0.0-snapshot-05749d0.jar.sha1 b/core/licenses/lucene-spatial-extras-7.0.0-snapshot-05749d0.jar.sha1 deleted file mode 100644 index 2d94742bc600c..0000000000000 --- a/core/licenses/lucene-spatial-extras-7.0.0-snapshot-05749d0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -6bb7d4f0270eb69ff01d13cc34ac75fd4451ec9e \ No newline at end of file diff --git a/core/licenses/lucene-spatial-extras-7.0.0-snapshot-89f6d17.jar.sha1 b/core/licenses/lucene-spatial-extras-7.0.0-snapshot-89f6d17.jar.sha1 new file mode 100644 index 0000000000000..ee3c72928820c --- /dev/null +++ b/core/licenses/lucene-spatial-extras-7.0.0-snapshot-89f6d17.jar.sha1 @@ -0,0 +1 @@ +a2c13be0fe4c5a98a30ec6ae673be1442409817c \ No newline at end of file diff --git a/core/licenses/lucene-spatial3d-7.0.0-snapshot-05749d0.jar.sha1 b/core/licenses/lucene-spatial3d-7.0.0-snapshot-05749d0.jar.sha1 deleted file mode 100644 index 49b43a35c4ce2..0000000000000 --- a/core/licenses/lucene-spatial3d-7.0.0-snapshot-05749d0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -fbf9869d1820bf7a6756e05f02467b140b30ff12 \ No newline at end of file diff --git a/core/licenses/lucene-spatial3d-7.0.0-snapshot-89f6d17.jar.sha1 b/core/licenses/lucene-spatial3d-7.0.0-snapshot-89f6d17.jar.sha1 new file mode 100644 index 0000000000000..28fab4a3819fb --- /dev/null +++ b/core/licenses/lucene-spatial3d-7.0.0-snapshot-89f6d17.jar.sha1 @@ -0,0 +1 @@ +92b8282e474845fdae31f9f239f953bc7164401f \ No newline at end of file diff --git a/core/licenses/lucene-suggest-7.0.0-snapshot-05749d0.jar.sha1 b/core/licenses/lucene-suggest-7.0.0-snapshot-05749d0.jar.sha1 deleted file mode 100644 index 9f39e5dd8f61c..0000000000000 --- a/core/licenses/lucene-suggest-7.0.0-snapshot-05749d0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -af8c10761e1b7186882613dce5f714c888e50371 \ No newline at end of file diff --git a/core/licenses/lucene-suggest-7.0.0-snapshot-89f6d17.jar.sha1 b/core/licenses/lucene-suggest-7.0.0-snapshot-89f6d17.jar.sha1 new file mode 100644 index 0000000000000..e928966f2746c --- /dev/null +++ b/core/licenses/lucene-suggest-7.0.0-snapshot-89f6d17.jar.sha1 @@ -0,0 +1 @@ +1c4aaea267ed41657ebf01769bfddbcab5b27414 \ No newline at end of file diff --git a/core/src/main/resources/org/elasticsearch/bootstrap/security.policy b/core/src/main/resources/org/elasticsearch/bootstrap/security.policy index fa8099ead2336..6d28944680ea4 100644 --- a/core/src/main/resources/org/elasticsearch/bootstrap/security.policy +++ b/core/src/main/resources/org/elasticsearch/bootstrap/security.policy @@ -31,7 +31,7 @@ grant codeBase "${codebase.securesm-1.1.jar}" { //// Very special jar permissions: //// These are dangerous permissions that we don't want to grant to everything. -grant codeBase "${codebase.lucene-core-7.0.0-snapshot-05749d0.jar}" { +grant codeBase "${codebase.lucene-core-7.0.0-snapshot-89f6d17.jar}" { // needed to allow MMapDirectory's "unmap hack" (die unmap hack, die) // java 8 package permission java.lang.RuntimePermission "accessClassInPackage.sun.misc"; @@ -42,7 +42,7 @@ grant codeBase "${codebase.lucene-core-7.0.0-snapshot-05749d0.jar}" { permission java.lang.RuntimePermission "accessDeclaredMembers"; }; -grant codeBase "${codebase.lucene-misc-7.0.0-snapshot-05749d0.jar}" { +grant codeBase "${codebase.lucene-misc-7.0.0-snapshot-89f6d17.jar}" { // needed to allow shard shrinking to use hard-links if possible via lucenes HardlinkCopyDirectoryWrapper permission java.nio.file.LinkPermission "hard"; }; diff --git a/core/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy b/core/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy index 8c9655c89e66b..9a089ef2810ff 100644 --- a/core/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy +++ b/core/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy @@ -33,7 +33,7 @@ grant codeBase "${codebase.securemock-1.2.jar}" { permission java.lang.reflect.ReflectPermission "suppressAccessChecks"; }; -grant codeBase "${codebase.lucene-test-framework-7.0.0-snapshot-05749d0.jar}" { +grant codeBase "${codebase.lucene-test-framework-7.0.0-snapshot-89f6d17.jar}" { // needed by RamUsageTester permission java.lang.reflect.ReflectPermission "suppressAccessChecks"; // needed for testing hardlinks in StoreRecoveryTests since we install MockFS diff --git a/modules/lang-expression/licenses/lucene-expressions-7.0.0-snapshot-05749d0.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-7.0.0-snapshot-05749d0.jar.sha1 deleted file mode 100644 index fa698133e4761..0000000000000 --- a/modules/lang-expression/licenses/lucene-expressions-7.0.0-snapshot-05749d0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -28c9991d67a24c4a72103cbdcaa1bd560adc67d4 \ No newline at end of file diff --git a/modules/lang-expression/licenses/lucene-expressions-7.0.0-snapshot-89f6d17.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-7.0.0-snapshot-89f6d17.jar.sha1 new file mode 100644 index 0000000000000..c47ca7d206cf1 --- /dev/null +++ b/modules/lang-expression/licenses/lucene-expressions-7.0.0-snapshot-89f6d17.jar.sha1 @@ -0,0 +1 @@ +52bc12bbc30db614056896495f30699d69eabae4 \ No newline at end of file diff --git a/plugins/analysis-icu/licenses/lucene-analyzers-icu-7.0.0-snapshot-05749d0.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analyzers-icu-7.0.0-snapshot-05749d0.jar.sha1 deleted file mode 100644 index 50a81a3345b83..0000000000000 --- a/plugins/analysis-icu/licenses/lucene-analyzers-icu-7.0.0-snapshot-05749d0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -c71806331eb8c6f690196eac4ec7f038bc027512 \ No newline at end of file diff --git a/plugins/analysis-icu/licenses/lucene-analyzers-icu-7.0.0-snapshot-89f6d17.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analyzers-icu-7.0.0-snapshot-89f6d17.jar.sha1 new file mode 100644 index 0000000000000..1ac5e494f00a7 --- /dev/null +++ b/plugins/analysis-icu/licenses/lucene-analyzers-icu-7.0.0-snapshot-89f6d17.jar.sha1 @@ -0,0 +1 @@ +e6e6d743c100e4d7bc55480d5333d634e41856ca \ No newline at end of file diff --git a/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-7.0.0-snapshot-05749d0.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-7.0.0-snapshot-05749d0.jar.sha1 deleted file mode 100644 index 93290a8ed304e..0000000000000 --- a/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-7.0.0-snapshot-05749d0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -fcc86339e16d85728f2becc2b404aeb69c2fd9fc \ No newline at end of file diff --git a/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-7.0.0-snapshot-89f6d17.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-7.0.0-snapshot-89f6d17.jar.sha1 new file mode 100644 index 0000000000000..8be206e2c1a65 --- /dev/null +++ b/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-7.0.0-snapshot-89f6d17.jar.sha1 @@ -0,0 +1 @@ +e520a9c7f8a2fc9f7c575940d9b24834a592ca25 \ No newline at end of file diff --git a/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-7.0.0-snapshot-05749d0.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-7.0.0-snapshot-05749d0.jar.sha1 deleted file mode 100644 index 191344494dc03..0000000000000 --- a/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-7.0.0-snapshot-05749d0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -6f3abd78bae787482b256e7629ee6c765546d476 \ No newline at end of file diff --git a/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-7.0.0-snapshot-89f6d17.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-7.0.0-snapshot-89f6d17.jar.sha1 new file mode 100644 index 0000000000000..2b69050029b47 --- /dev/null +++ b/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-7.0.0-snapshot-89f6d17.jar.sha1 @@ -0,0 +1 @@ +32fe29deb1c10cb7ae70d5f4d95fcc414f9813d6 \ No newline at end of file diff --git a/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-7.0.0-snapshot-05749d0.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-7.0.0-snapshot-05749d0.jar.sha1 deleted file mode 100644 index 6c9093a23d9c0..0000000000000 --- a/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-7.0.0-snapshot-05749d0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -0311b02c31c634cfefaf51b551edb896d73840e2 \ No newline at end of file diff --git a/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-7.0.0-snapshot-89f6d17.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-7.0.0-snapshot-89f6d17.jar.sha1 new file mode 100644 index 0000000000000..5412498f1c867 --- /dev/null +++ b/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-7.0.0-snapshot-89f6d17.jar.sha1 @@ -0,0 +1 @@ +e50f4ab0d6ebf85b282a86707309343e3260c4a2 \ No newline at end of file diff --git a/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-7.0.0-snapshot-05749d0.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-7.0.0-snapshot-05749d0.jar.sha1 deleted file mode 100644 index 501140d4149bb..0000000000000 --- a/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-7.0.0-snapshot-05749d0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -d511a9ef81c2f80361d2f86f8848d6cd504b390e \ No newline at end of file diff --git a/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-7.0.0-snapshot-89f6d17.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-7.0.0-snapshot-89f6d17.jar.sha1 new file mode 100644 index 0000000000000..c845088ffb7b1 --- /dev/null +++ b/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-7.0.0-snapshot-89f6d17.jar.sha1 @@ -0,0 +1 @@ +a41c22ef7dd43991e1f3555ff527ac79eb47fdca \ No newline at end of file diff --git a/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-7.0.0-snapshot-05749d0.jar.sha1 b/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-7.0.0-snapshot-05749d0.jar.sha1 deleted file mode 100644 index f21f6e197f440..0000000000000 --- a/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-7.0.0-snapshot-05749d0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -7f0157bf06682dda0d0e0a2120403099ff9f8a36 \ No newline at end of file diff --git a/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-7.0.0-snapshot-89f6d17.jar.sha1 b/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-7.0.0-snapshot-89f6d17.jar.sha1 new file mode 100644 index 0000000000000..973047cdf8986 --- /dev/null +++ b/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-7.0.0-snapshot-89f6d17.jar.sha1 @@ -0,0 +1 @@ +01fe11b45d9f6a68ef1e9994bebd81d26632efc5 \ No newline at end of file From 8b3ad108682d5ebf75cf0eeadd9f6e5137934155 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Thu, 13 Apr 2017 10:59:30 +0200 Subject: [PATCH 83/87] Fix compile errors. --- .../grouping/CollapsingDocValuesSource.java | 71 ++++++++++++------- .../grouping/CollapsingTopDocsCollector.java | 65 ++--------------- 2 files changed, 54 insertions(+), 82 deletions(-) diff --git a/core/src/main/java/org/apache/lucene/search/grouping/CollapsingDocValuesSource.java b/core/src/main/java/org/apache/lucene/search/grouping/CollapsingDocValuesSource.java index 2f805e08c2d6d..e48773389021c 100644 --- a/core/src/main/java/org/apache/lucene/search/grouping/CollapsingDocValuesSource.java +++ b/core/src/main/java/org/apache/lucene/search/grouping/CollapsingDocValuesSource.java @@ -22,32 +22,32 @@ import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.LeafReader; +import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.NumericDocValues; import org.apache.lucene.index.SortedDocValues; import org.apache.lucene.index.SortedNumericDocValues; import org.apache.lucene.index.SortedSetDocValues; -import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.BytesRef; import org.elasticsearch.index.fielddata.AbstractNumericDocValues; import org.elasticsearch.index.fielddata.AbstractSortedDocValues; import java.io.IOException; +import java.util.Collection; /** * Utility class that ensures that a single collapse key is extracted per document. */ -abstract class CollapsingDocValuesSource { +abstract class CollapsingDocValuesSource extends GroupSelector { protected final String field; CollapsingDocValuesSource(String field) throws IOException { this.field = field; } - abstract T get(int doc) throws IOException; - - abstract T copy(T value, T reuse); - - abstract void setNextReader(LeafReader reader) throws IOException; + @Override + public void setGroups(Collection> groups) { + throw new UnsupportedOperationException(); + } /** * Implementation for {@link NumericDocValues} and {@link SortedNumericDocValues}. @@ -55,27 +55,38 @@ abstract class CollapsingDocValuesSource { */ static class Numeric extends CollapsingDocValuesSource { private NumericDocValues values; + private long value; + private boolean hasValue; Numeric(String field) throws IOException { super(field); } @Override - public Long get(int doc) throws IOException { + public State advanceTo(int doc) throws IOException { if (values.advanceExact(doc)) { - return values.longValue(); + hasValue = true; + value = values.longValue(); + return State.ACCEPT; } else { - return null; + hasValue = false; + return State.SKIP; } } @Override - public Long copy(Long value, Long reuse) { - return value; + public Long currentValue() { + return hasValue ? value : null; + } + + @Override + public Long copyValue() { + return currentValue(); } @Override - public void setNextReader(LeafReader reader) throws IOException { + public void setNextReader(LeafReaderContext readerContext) throws IOException { + LeafReader reader = readerContext.reader(); DocValuesType type = getDocValuesType(reader, field); if (type == null || type == DocValuesType.NONE) { values = DocValues.emptyNumeric(); @@ -135,38 +146,50 @@ public long longValue() throws IOException { */ static class Keyword extends CollapsingDocValuesSource { private SortedDocValues values; + private int ord; Keyword(String field) throws IOException { super(field); } @Override - public BytesRef get(int doc) throws IOException { + public org.apache.lucene.search.grouping.GroupSelector.State advanceTo(int doc) + throws IOException { if (values.advanceExact(doc)) { - return values.binaryValue(); + ord = values.ordValue(); + return State.ACCEPT; } else { + ord = -1; + return State.SKIP; + } + } + + @Override + public BytesRef currentValue() { + if (ord == -1) { return null; + } else { + try { + return values.lookupOrd(ord); + } catch (IOException e) { + throw new RuntimeException(e); + } } } @Override - public BytesRef copy(BytesRef value, BytesRef reuse) { + public BytesRef copyValue() { + BytesRef value = currentValue(); if (value == null) { return null; - } - if (reuse != null) { - reuse.bytes = ArrayUtil.grow(reuse.bytes, value.length); - reuse.offset = 0; - reuse.length = value.length; - System.arraycopy(value.bytes, value.offset, reuse.bytes, 0, value.length); - return reuse; } else { return BytesRef.deepCopyOf(value); } } @Override - public void setNextReader(LeafReader reader) throws IOException { + public void setNextReader(LeafReaderContext readerContext) throws IOException { + LeafReader reader = readerContext.reader(); DocValuesType type = getDocValuesType(reader, field); if (type == null || type == DocValuesType.NONE) { values = DocValues.emptySorted(); diff --git a/core/src/main/java/org/apache/lucene/search/grouping/CollapsingTopDocsCollector.java b/core/src/main/java/org/apache/lucene/search/grouping/CollapsingTopDocsCollector.java index 33d1b94685a90..b5cb02bcd6536 100644 --- a/core/src/main/java/org/apache/lucene/search/grouping/CollapsingTopDocsCollector.java +++ b/core/src/main/java/org/apache/lucene/search/grouping/CollapsingTopDocsCollector.java @@ -18,13 +18,11 @@ */ package org.apache.lucene.search.grouping; -import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.search.FieldDoc; import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.Scorer; import org.apache.lucene.search.Sort; import org.apache.lucene.search.SortField; -import org.apache.lucene.util.BytesRef; import java.io.IOException; import java.util.Collection; @@ -37,7 +35,7 @@ * output. The collapsing is done in a single pass by selecting only the top sorted document per collapse key. * The value used for the collapse key of each group can be found in {@link CollapseTopFieldDocs#collapseValues}. */ -public abstract class CollapsingTopDocsCollector extends FirstPassGroupingCollector { +public final class CollapsingTopDocsCollector extends FirstPassGroupingCollector { protected final String collapseField; protected final Sort sort; @@ -47,9 +45,9 @@ public abstract class CollapsingTopDocsCollector extends FirstPassGroupingCol private float maxScore; private final boolean trackMaxScore; - private CollapsingTopDocsCollector(String collapseField, Sort sort, + CollapsingTopDocsCollector(GroupSelector groupSelector, String collapseField, Sort sort, int topN, boolean trackMaxScore) throws IOException { - super(sort, topN); + super(groupSelector, sort, topN); this.collapseField = collapseField; this.trackMaxScore = trackMaxScore; if (trackMaxScore) { @@ -121,57 +119,6 @@ public void collect(int doc) throws IOException { totalHitCount++; } - private static class Numeric extends CollapsingTopDocsCollector { - private final CollapsingDocValuesSource.Numeric source; - - private Numeric(String collapseField, Sort sort, int topN, boolean trackMaxScore) throws IOException { - super(collapseField, sort, topN, trackMaxScore); - source = new CollapsingDocValuesSource.Numeric(collapseField); - } - - @Override - protected void doSetNextReader(LeafReaderContext readerContext) throws IOException { - super.doSetNextReader(readerContext); - source.setNextReader(readerContext.reader()); - } - - @Override - protected Long getDocGroupValue(int doc) throws IOException { - return source.get(doc); - } - - @Override - protected Long copyDocGroupValue(Long groupValue, Long reuse) { - return source.copy(groupValue, reuse); - } - } - - private static class Keyword extends CollapsingTopDocsCollector { - private final CollapsingDocValuesSource.Keyword source; - - private Keyword(String collapseField, Sort sort, int topN, boolean trackMaxScore) throws IOException { - super(collapseField, sort, topN, trackMaxScore); - source = new CollapsingDocValuesSource.Keyword(collapseField); - - } - - @Override - protected void doSetNextReader(LeafReaderContext readerContext) throws IOException { - super.doSetNextReader(readerContext); - source.setNextReader(readerContext.reader()); - } - - @Override - protected BytesRef getDocGroupValue(int doc) throws IOException { - return source.get(doc); - } - - @Override - protected BytesRef copyDocGroupValue(BytesRef groupValue, BytesRef reuse) { - return source.copy(groupValue, reuse); - } - } - /** * Create a collapsing top docs collector on a {@link org.apache.lucene.index.NumericDocValues} field. * It accepts also {@link org.apache.lucene.index.SortedNumericDocValues} field but @@ -189,7 +136,8 @@ protected BytesRef copyDocGroupValue(BytesRef groupValue, BytesRef reuse) { */ public static CollapsingTopDocsCollector createNumeric(String collapseField, Sort sort, int topN, boolean trackMaxScore) throws IOException { - return new Numeric(collapseField, sort, topN, trackMaxScore); + return new CollapsingTopDocsCollector<>(new CollapsingDocValuesSource.Numeric(collapseField), + collapseField, sort, topN, trackMaxScore); } /** @@ -208,7 +156,8 @@ public static CollapsingTopDocsCollector createNumeric(String collapseField, */ public static CollapsingTopDocsCollector createKeyword(String collapseField, Sort sort, int topN, boolean trackMaxScore) throws IOException { - return new Keyword(collapseField, sort, topN, trackMaxScore); + return new CollapsingTopDocsCollector<>(new CollapsingDocValuesSource.Keyword(collapseField), + collapseField, sort, topN, trackMaxScore); } } From 917804666c0c664b80e17fae505fb2f7cd6e72cc Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Thu, 13 Apr 2017 11:11:54 +0200 Subject: [PATCH 84/87] Remove AwaitsFix. --- .../org/elasticsearch/common/lucene/ShardCoreKeyMapTests.java | 1 - 1 file changed, 1 deletion(-) diff --git a/core/src/test/java/org/elasticsearch/common/lucene/ShardCoreKeyMapTests.java b/core/src/test/java/org/elasticsearch/common/lucene/ShardCoreKeyMapTests.java index 515d62140d2e6..ca24a1346fe11 100644 --- a/core/src/test/java/org/elasticsearch/common/lucene/ShardCoreKeyMapTests.java +++ b/core/src/test/java/org/elasticsearch/common/lucene/ShardCoreKeyMapTests.java @@ -57,7 +57,6 @@ public void testMissingShard() throws IOException { } } - @AwaitsFix(bugUrl="https://issues.apache.org/jira/browse/LUCENE-7781") public void testAddingAClosedReader() throws Exception { LeafReader reader; try (Directory dir = newDirectory(); From 945e95583a257e23498213e0ad6d6863b1e2ead9 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Thu, 13 Apr 2017 13:51:27 +0200 Subject: [PATCH 85/87] Fix docs expectations. --- .../metrics/geocentroid-aggregation.asciidoc | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/reference/aggregations/metrics/geocentroid-aggregation.asciidoc b/docs/reference/aggregations/metrics/geocentroid-aggregation.asciidoc index 89aa091bba2f0..4c9686fa1e4bc 100644 --- a/docs/reference/aggregations/metrics/geocentroid-aggregation.asciidoc +++ b/docs/reference/aggregations/metrics/geocentroid-aggregation.asciidoc @@ -60,8 +60,8 @@ The response for the above aggregation: "aggregations": { "centroid": { "location": { - "lat": 51.009829603135586, - "lon": 3.966213036328554 + "lat": 51.00982963107526, + "lon": 3.9662130922079086 } } } @@ -113,7 +113,7 @@ The response for the above aggregation: "centroid": { "location": { "lat": 52.371655656024814, - "lon": 4.909563269466162 + "lon": 4.909563297405839 } } }, @@ -122,8 +122,8 @@ The response for the above aggregation: "doc_count": 2, "centroid": { "location": { - "lat": 48.86055544484407, - "lon": 2.331694420427084 + "lat": 48.86055548675358, + "lon": 2.3316944623366 } } }, @@ -132,8 +132,8 @@ The response for the above aggregation: "doc_count": 1, "centroid": { "location": { - "lat": 51.222899928689, - "lon": 4.405199903994799 + "lat": 51.22289997059852, + "lon": 4.40519998781383 } } } From 9426a70f624cc76b3f5456e3cc486a30116a0e1b Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Thu, 13 Apr 2017 17:00:50 +0200 Subject: [PATCH 86/87] Minor cleanups. --- .../index/fielddata/FieldData.java | 45 ++------- .../index/fielddata/NumericDoubleValues.java | 36 ++----- .../ordinals/SinglePackedOrdinals.java | 2 +- .../AbstractGeoPointDVIndexFieldData.java | 83 ---------------- .../plain/BytesBinaryDVAtomicFieldData.java | 36 ++----- .../plain/GeoPointDVAtomicFieldData.java | 98 ------------------- .../index/mapper/IpFieldMapper.java | 39 +++----- 7 files changed, 35 insertions(+), 304 deletions(-) delete mode 100644 core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractGeoPointDVIndexFieldData.java delete mode 100644 core/src/main/java/org/elasticsearch/index/fielddata/plain/GeoPointDVAtomicFieldData.java diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/FieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/FieldData.java index 76ac003f99a03..a1ac341b56f95 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/FieldData.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/FieldData.java @@ -439,7 +439,7 @@ public boolean advanceExact(int doc) throws IOException { public int docValueCount() { return count; } - + @Override public BytesRef nextValue() throws IOException { return values.lookupOrd(values.nextOrd()); @@ -551,9 +551,10 @@ public int docValueCount() { } - private static class LongCastedValues extends NumericDocValues { + private static class LongCastedValues extends AbstractNumericDocValues { private final NumericDoubleValues values; + private int docID = -1; LongCastedValues(NumericDoubleValues values) { this.values = values; @@ -561,6 +562,7 @@ private static class LongCastedValues extends NumericDocValues { @Override public boolean advanceExact(int target) throws IOException { + docID = target; return values.advanceExact(target); } @@ -571,26 +573,11 @@ public long longValue() throws IOException { @Override public int docID() { - throw new UnsupportedOperationException(); - } - - @Override - public int nextDoc() throws IOException { - throw new UnsupportedOperationException(); - } - - @Override - public int advance(int target) throws IOException { - throw new UnsupportedOperationException(); - } - - @Override - public long cost() { - throw new UnsupportedOperationException(); + return docID; } } - private static class SortedLongCastedValues extends SortedNumericDocValues { + private static class SortedLongCastedValues extends AbstractSortedNumericDocValues { private final SortedNumericDoubleValues values; @@ -613,25 +600,5 @@ public long nextValue() throws IOException { return (long) values.nextValue(); } - @Override - public int docID() { - throw new UnsupportedOperationException(); - } - - @Override - public int nextDoc() throws IOException { - throw new UnsupportedOperationException(); - } - - @Override - public int advance(int target) throws IOException { - throw new UnsupportedOperationException(); - } - - @Override - public long cost() { - throw new UnsupportedOperationException(); - } - } } diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/NumericDoubleValues.java b/core/src/main/java/org/elasticsearch/index/fielddata/NumericDoubleValues.java index dd84ee14b89c6..0d78d3c7905d1 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/NumericDoubleValues.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/NumericDoubleValues.java @@ -36,9 +36,11 @@ protected NumericDoubleValues() {} // TODO: this interaction with sort comparators is really ugly... /** Returns numeric docvalues view of raw double bits */ public NumericDocValues getRawDoubleValues() { - return new NumericDocValues() { + return new AbstractNumericDocValues() { + private int docID = -1; @Override public boolean advanceExact(int target) throws IOException { + docID = target; return NumericDoubleValues.this.advanceExact(target); } @Override @@ -47,19 +49,7 @@ public long longValue() throws IOException { } @Override public int docID() { - throw new UnsupportedOperationException(); - } - @Override - public int nextDoc() throws IOException { - throw new UnsupportedOperationException(); - } - @Override - public int advance(int target) throws IOException { - throw new UnsupportedOperationException(); - } - @Override - public long cost() { - throw new UnsupportedOperationException(); + return docID; } }; } @@ -67,9 +57,11 @@ public long cost() { // yes... this is doing what the previous code was doing... /** Returns numeric docvalues view of raw float bits */ public NumericDocValues getRawFloatValues() { - return new NumericDocValues() { + return new AbstractNumericDocValues() { + private int docID = -1; @Override public boolean advanceExact(int target) throws IOException { + docID = target; return NumericDoubleValues.this.advanceExact(target); } @Override @@ -78,19 +70,7 @@ public long longValue() throws IOException { } @Override public int docID() { - throw new UnsupportedOperationException(); - } - @Override - public int nextDoc() throws IOException { - throw new UnsupportedOperationException(); - } - @Override - public int advance(int target) throws IOException { - throw new UnsupportedOperationException(); - } - @Override - public long cost() { - throw new UnsupportedOperationException(); + return docID; } }; } diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/SinglePackedOrdinals.java b/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/SinglePackedOrdinals.java index a7d93693e89fa..27f0aadee8719 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/SinglePackedOrdinals.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/SinglePackedOrdinals.java @@ -67,7 +67,7 @@ private static class Docs extends AbstractSortedDocValues { private final int maxOrd; private final PackedInts.Reader reader; private final ValuesHolder values; - + private int currentDoc = -1; private int currentOrd; diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractGeoPointDVIndexFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractGeoPointDVIndexFieldData.java deleted file mode 100644 index 8db38e59ce02f..0000000000000 --- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractGeoPointDVIndexFieldData.java +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.fielddata.plain; - -import org.apache.lucene.index.DocValues; -import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.search.SortField; -import org.elasticsearch.common.Nullable; -import org.elasticsearch.index.Index; -import org.elasticsearch.index.IndexSettings; -import org.elasticsearch.index.fielddata.AtomicGeoPointFieldData; -import org.elasticsearch.index.fielddata.IndexFieldData; -import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested; -import org.elasticsearch.index.fielddata.IndexFieldDataCache; -import org.elasticsearch.index.fielddata.IndexGeoPointFieldData; -import org.elasticsearch.index.mapper.MappedFieldType; -import org.elasticsearch.index.mapper.MapperService; -import org.elasticsearch.indices.breaker.CircuitBreakerService; -import org.elasticsearch.search.MultiValueMode; - -import java.io.IOException; - -public abstract class AbstractGeoPointDVIndexFieldData extends DocValuesIndexFieldData implements IndexGeoPointFieldData { - - AbstractGeoPointDVIndexFieldData(Index index, String fieldName) { - super(index, fieldName); - } - - @Override - public SortField sortField(@Nullable Object missingValue, MultiValueMode sortMode, Nested nested, boolean reverse) { - throw new IllegalArgumentException("can't sort on geo_point field without using specific sorting feature, like geo_distance"); - } - - /** - * Lucene 5.4 GeoPointFieldType - */ - public static class GeoPointDVIndexFieldData extends AbstractGeoPointDVIndexFieldData { - - public GeoPointDVIndexFieldData(Index index, String fieldName) { - super(index, fieldName); - } - - @Override - public AtomicGeoPointFieldData load(LeafReaderContext context) { - try { - return new GeoPointDVAtomicFieldData(DocValues.getSortedNumeric(context.reader(), fieldName)); - } catch (IOException e) { - throw new IllegalStateException("Cannot load doc values", e); - } - } - - @Override - public AtomicGeoPointFieldData loadDirect(LeafReaderContext context) throws Exception { - return load(context); - } - } - - public static class Builder implements IndexFieldData.Builder { - @Override - public IndexFieldData build(IndexSettings indexSettings, MappedFieldType fieldType, IndexFieldDataCache cache, - CircuitBreakerService breakerService, MapperService mapperService) { - // Ignore breaker - return new GeoPointDVIndexFieldData(indexSettings.getIndex(), fieldType.name()); - } - } -} diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/BytesBinaryDVAtomicFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/BytesBinaryDVAtomicFieldData.java index b42cfefc46ab0..aa2775046ff1b 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/BytesBinaryDVAtomicFieldData.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/plain/BytesBinaryDVAtomicFieldData.java @@ -22,16 +22,12 @@ import org.apache.lucene.index.BinaryDocValues; import org.apache.lucene.store.ByteArrayDataInput; import org.apache.lucene.util.Accountable; -import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.BytesRefBuilder; -import org.apache.lucene.util.RamUsageEstimator; import org.elasticsearch.index.fielddata.AtomicFieldData; import org.elasticsearch.index.fielddata.ScriptDocValues; import org.elasticsearch.index.fielddata.SortedBinaryDocValues; import java.io.IOException; -import java.util.Arrays; import java.util.Collection; import java.util.Collections; @@ -59,36 +55,17 @@ public SortedBinaryDocValues getBytesValues() { return new SortedBinaryDocValues() { int count; - BytesRefBuilder[] refs = new BytesRefBuilder[0]; - int refsCursor; final ByteArrayDataInput in = new ByteArrayDataInput(); + final BytesRef scratch = new BytesRef(); @Override public boolean advanceExact(int doc) throws IOException { - refsCursor = 0; if (values.advanceExact(doc)) { final BytesRef bytes = values.binaryValue(); + assert bytes.length > 0; in.reset(bytes.bytes, bytes.offset, bytes.length); - if (bytes.length == 0) { - count = 0; - } else { - count = in.readVInt(); - if (count > refs.length) { - final int previousLength = refs.length; - refs = Arrays.copyOf(refs, ArrayUtil.oversize(count, - RamUsageEstimator.NUM_BYTES_OBJECT_REF)); - for (int i = previousLength; i < refs.length; ++i) { - refs[i] = new BytesRefBuilder(); - } - } - for (int i = 0; i < count; ++i) { - final int length = in.readVInt(); - final BytesRefBuilder scratch = refs[i]; - scratch.grow(length); - in.readBytes(scratch.bytes(), 0, length); - scratch.setLength(length); - } - } + count = in.readVInt(); + scratch.bytes = bytes.bytes; return true; } else { return false; @@ -102,7 +79,10 @@ public int docValueCount() { @Override public BytesRef nextValue() throws IOException { - return refs[refsCursor++].get(); + scratch.length = in.readVInt(); + scratch.offset = in.getPosition(); + in.setPosition(scratch.offset + scratch.length); + return scratch; } }; diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/GeoPointDVAtomicFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/GeoPointDVAtomicFieldData.java deleted file mode 100644 index 11cb1c7b8af71..0000000000000 --- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/GeoPointDVAtomicFieldData.java +++ /dev/null @@ -1,98 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.fielddata.plain; - -import org.apache.lucene.index.SortedNumericDocValues; -import org.apache.lucene.util.Accountable; -import org.apache.lucene.util.ArrayUtil; -import org.apache.lucene.util.RamUsageEstimator; -import org.elasticsearch.common.geo.GeoPoint; -import org.elasticsearch.index.fielddata.MultiGeoPointValues; - -import java.io.IOException; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; - -final class GeoPointDVAtomicFieldData extends AbstractAtomicGeoPointFieldData { - - private final SortedNumericDocValues values; - - GeoPointDVAtomicFieldData(SortedNumericDocValues values) { - super(); - this.values = values; - } - - @Override - public long ramBytesUsed() { - return 0; // not exposed by Lucene - } - - @Override - public Collection getChildResources() { - return Collections.emptyList(); - } - - @Override - public void close() { - // no-op - } - - @Override - public MultiGeoPointValues getGeoPointValues() { - return new MultiGeoPointValues() { - GeoPoint[] points = new GeoPoint[0]; - private int pointsCursor = 0; - private int count = 0; - - @Override - public boolean advanceExact(int doc) throws IOException { - if (values.advanceExact(doc)) { - count = values.docValueCount(); - pointsCursor = 0; - if (count > points.length) { - final int previousLength = points.length; - points = Arrays.copyOf(points, - ArrayUtil.oversize(count, RamUsageEstimator.NUM_BYTES_OBJECT_REF)); - for (int i = previousLength; i < points.length; ++i) { - points[i] = new GeoPoint(Double.NaN, Double.NaN); - } - } - for (int i = 0; i < count; ++i) { - points[i].resetFromIndexHash(values.nextValue()); - } - return true; - } else { - return false; - } - } - - @Override - public int docValueCount() { - return count; - } - - @Override - public GeoPoint nextValue() { - return points[pointsCursor++]; - } - }; - } -} \ No newline at end of file diff --git a/core/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java index 5dfa0a9d04757..3d6f4efba1405 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java @@ -30,6 +30,7 @@ import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.Query; +import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.BytesRef; import org.elasticsearch.action.fieldstats.FieldStats; import org.elasticsearch.common.Explicit; @@ -233,35 +234,27 @@ public FieldStats.Ip stats(IndexReader reader) throws IOException { public static final class IpScriptDocValues extends ScriptDocValues { - private final SortedSetDocValues values; + private final SortedSetDocValues in; + private long[] ords = new long[0]; private int count; - private int lastIndex; - private int doc; - public IpScriptDocValues(SortedSetDocValues values) { - this.values = values; + public IpScriptDocValues(SortedSetDocValues in) { + this.in = in; } @Override - public void setNextDocId(int docId) { + public void setNextDocId(int docId) throws IOException { count = 0; - try { - if (values.advanceExact(docId)) { - for (long ord = values.nextOrd(); ord != SortedSetDocValues.NO_MORE_ORDS; ord = values.nextOrd()) { - count++; - } - boolean hasValue = values.advanceExact(docId); - assert hasValue; - doc = docId; - lastIndex = -1; + if (in.advanceExact(docId)) { + for (long ord = in.nextOrd(); ord != SortedSetDocValues.NO_MORE_ORDS; ord = in.nextOrd()) { + ords = ArrayUtil.grow(ords, count + 1); + ords[count++] = ord; } - } catch (IOException e) { - throw new RuntimeException(e); } } public String getValue() { - if (isEmpty()) { + if (count == 0) { return null; } else { return get(0); @@ -271,15 +264,7 @@ public String getValue() { @Override public String get(int index) { try { - if (index <= lastIndex) { - boolean hasValue = values.advanceExact(doc); - assert hasValue; - lastIndex = -1; - } - for (int i = lastIndex + 1; i < index; ++i) { - values.nextOrd(); - } - BytesRef encoded = values.lookupOrd(values.nextOrd()); + BytesRef encoded = in.lookupOrd(ords[index]); InetAddress address = InetAddressPoint.decode( Arrays.copyOfRange(encoded.bytes, encoded.offset, encoded.offset + encoded.length)); return InetAddresses.toAddrString(address); From 15b8d065e27834288a83c580d7b95f8eccba34f5 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Tue, 18 Apr 2017 13:52:40 +0200 Subject: [PATCH 87/87] Fix wrong sharing of readers. --- .../lucene/uid/PerThreadIDVersionAndSeqNoLookup.java | 8 ++------ .../common/lucene/uid/VersionsAndSeqNoResolver.java | 2 +- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/common/lucene/uid/PerThreadIDVersionAndSeqNoLookup.java b/core/src/main/java/org/elasticsearch/common/lucene/uid/PerThreadIDVersionAndSeqNoLookup.java index d1cc1d2088ebc..96c9f30a9546f 100644 --- a/core/src/main/java/org/elasticsearch/common/lucene/uid/PerThreadIDVersionAndSeqNoLookup.java +++ b/core/src/main/java/org/elasticsearch/common/lucene/uid/PerThreadIDVersionAndSeqNoLookup.java @@ -50,9 +50,6 @@ final class PerThreadIDVersionAndSeqNoLookup { // TODO: do we really need to store all this stuff? some if it might not speed up anything. // we keep it around for now, to reduce the amount of e.g. hash lookups by field and stuff - /** The segment reader. */ - private final LeafReader reader; - /** terms enum for uid field */ private final TermsEnum termsEnum; @@ -66,7 +63,6 @@ final class PerThreadIDVersionAndSeqNoLookup { * Initialize lookup for the provided segment */ PerThreadIDVersionAndSeqNoLookup(LeafReader reader) throws IOException { - this.reader = reader; Fields fields = reader.fields(); Terms terms = fields.terms(UidFieldMapper.NAME); termsEnum = terms.iterator(); @@ -130,7 +126,7 @@ DocIdAndSeqNo lookupSeqNo(BytesRef id, Bits liveDocs, LeafReaderContext context) "context's reader is not the same as the reader class was initialized on."; int docID = getDocID(id, liveDocs); if (docID != DocIdSetIterator.NO_MORE_DOCS) { - NumericDocValues seqNos = reader.getNumericDocValues(SeqNoFieldMapper.NAME); + NumericDocValues seqNos = context.reader().getNumericDocValues(SeqNoFieldMapper.NAME); long seqNo; if (seqNos != null && seqNos.advanceExact(docID)) { seqNo = seqNos.longValue(); @@ -148,7 +144,7 @@ DocIdAndSeqNo lookupSeqNo(BytesRef id, Bits liveDocs, LeafReaderContext context) * * Note that 0 is an illegal primary term. See {@link org.elasticsearch.cluster.metadata.IndexMetaData#primaryTerm(int)} **/ - long lookUpPrimaryTerm(int docID) throws IOException { + long lookUpPrimaryTerm(int docID, LeafReader reader) throws IOException { NumericDocValues primaryTerms = reader.getNumericDocValues(SeqNoFieldMapper.PRIMARY_TERM_NAME); if (primaryTerms != null && primaryTerms.advanceExact(docID)) { return primaryTerms.longValue(); diff --git a/core/src/main/java/org/elasticsearch/common/lucene/uid/VersionsAndSeqNoResolver.java b/core/src/main/java/org/elasticsearch/common/lucene/uid/VersionsAndSeqNoResolver.java index 59930c029c6d2..409ce8dec29be 100644 --- a/core/src/main/java/org/elasticsearch/common/lucene/uid/VersionsAndSeqNoResolver.java +++ b/core/src/main/java/org/elasticsearch/common/lucene/uid/VersionsAndSeqNoResolver.java @@ -160,7 +160,7 @@ public static DocIdAndSeqNo loadDocIdAndSeqNo(IndexReader reader, Term term) thr public static long loadPrimaryTerm(DocIdAndSeqNo docIdAndSeqNo) throws IOException { LeafReader leaf = docIdAndSeqNo.context.reader(); PerThreadIDVersionAndSeqNoLookup lookup = getLookupState(leaf); - long result = lookup.lookUpPrimaryTerm(docIdAndSeqNo.docId); + long result = lookup.lookUpPrimaryTerm(docIdAndSeqNo.docId, leaf); assert result > 0 : "should always resolve a primary term for a resolved sequence number. primary_term [" + result + "]" + " docId [" + docIdAndSeqNo.docId + "] seqNo [" + docIdAndSeqNo.seqNo + "]"; return result;