From d808e751f41c9c8fd20c38bbcfa218fadefd267c Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Wed, 15 Mar 2017 15:14:32 -0700 Subject: [PATCH 01/16] Mapping: Fix NPE with scaled floats stats when field is not indexed (#23528) This fixes an NPE in finding scaled float stats. The type of min/max methods on the wrapped long stats returns a boxed type, but in the case this is null, the unbox done for the FieldStats.Double ctor primitive types will cause the NPE. These methods would have null for min/max when the field exists, but does not actually have points values. fixes #23487 --- .../index/mapper/ScaledFloatFieldMapper.java | 11 ++++++++--- .../index/mapper/ScaledFloatFieldTypeTests.java | 12 +++++++++++- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java index 8920207778e76..62ff8bdede08e 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java @@ -265,11 +265,16 @@ public FieldStats stats(IndexReader reader) throws IOException { if (stats == null) { return null; } - return new FieldStats.Double(stats.getMaxDoc(), stats.getDocCount(), + if (stats.hasMinMax()) { + return new FieldStats.Double(stats.getMaxDoc(), stats.getDocCount(), stats.getSumDocFreq(), stats.getSumTotalTermFreq(), stats.isSearchable(), stats.isAggregatable(), - stats.getMinValue() == null ? null : stats.getMinValue() / scalingFactor, - stats.getMaxValue() == null ? null : stats.getMaxValue() / scalingFactor); + stats.getMinValue() / scalingFactor, + stats.getMaxValue() / scalingFactor); + } + return new FieldStats.Double(stats.getMaxDoc(), stats.getDocCount(), + stats.getSumDocFreq(), stats.getSumTotalTermFreq(), + stats.isSearchable(), stats.isAggregatable()); } @Override diff --git a/core/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldTypeTests.java b/core/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldTypeTests.java index e1fdf10356444..dd66421986779 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldTypeTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldTypeTests.java @@ -23,6 +23,7 @@ import org.apache.lucene.document.DoublePoint; import org.apache.lucene.document.LongPoint; import org.apache.lucene.document.SortedNumericDocValuesField; +import org.apache.lucene.document.StoredField; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; @@ -143,6 +144,15 @@ public void testStats() throws IOException { assertNull(ft.stats(reader)); } Document doc = new Document(); + doc.add(new StoredField("scaled_float", -1)); + w.addDocument(doc); + try (DirectoryReader reader = DirectoryReader.open(w)) { + // field exists, but has no point values + FieldStats stats = ft.stats(reader); + assertFalse(stats.hasMinMax()); + assertNull(stats.getMinValue()); + assertNull(stats.getMaxValue()); + } LongPoint point = new LongPoint("scaled_float", -1); doc.add(point); w.addDocument(doc); @@ -152,7 +162,7 @@ public void testStats() throws IOException { FieldStats stats = ft.stats(reader); assertEquals(-1/ft.getScalingFactor(), stats.getMinValue()); assertEquals(10/ft.getScalingFactor(), stats.getMaxValue()); - assertEquals(2, stats.getMaxDoc()); + assertEquals(3, stats.getMaxDoc()); } w.deleteAll(); try (DirectoryReader reader = DirectoryReader.open(w)) { From cb16ed1e26bcea52bb40adfceb18604b91c0cd82 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Wed, 15 Mar 2017 15:43:07 -0700 Subject: [PATCH 02/16] Fix num docs to be positive in bucket deferring collector test --- .../aggregations/bucket/BestBucketsDeferringCollectorTests.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/BestBucketsDeferringCollectorTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/BestBucketsDeferringCollectorTests.java index 02cd88f16fa04..975c1a0d4664d 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/BestBucketsDeferringCollectorTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/BestBucketsDeferringCollectorTests.java @@ -46,7 +46,7 @@ public class BestBucketsDeferringCollectorTests extends AggregatorTestCase { public void testReplay() throws Exception { Directory directory = newDirectory(); RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory); - int numDocs = randomInt(128); + int numDocs = randomIntBetween(1, 128); int maxNumValues = randomInt(16); for (int i = 0; i < numDocs; i++) { Document document = new Document(); From b902ab9e893c0184c84fa49ab7dd45ea56d37915 Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Wed, 15 Mar 2017 16:33:46 -0700 Subject: [PATCH 03/16] Remove extra line from license header This commit removes an extra line from the license header on the file EvilBootstrapCheckTests.java. --- .../org/elasticsearch/bootstrap/EvilBootstrapChecksTests.java | 1 - 1 file changed, 1 deletion(-) diff --git a/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/EvilBootstrapChecksTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/EvilBootstrapChecksTests.java index 372ae3a8885a9..79c3f9248b5e1 100644 --- a/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/EvilBootstrapChecksTests.java +++ b/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/EvilBootstrapChecksTests.java @@ -15,7 +15,6 @@ * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. - * */ package org.elasticsearch.bootstrap; From 4c11ebc8b9e916987ac0cd77370d85380088c96d Mon Sep 17 00:00:00 2001 From: Jack Conradson Date: Wed, 15 Mar 2017 17:18:34 -0700 Subject: [PATCH 04/16] Fix example in documentation for Painless using _source. (#21322) --- .../search/request/script-fields.asciidoc | 22 ++++++++++--------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/docs/reference/search/request/script-fields.asciidoc b/docs/reference/search/request/script-fields.asciidoc index b544c79e4f202..b8132db7aadf1 100644 --- a/docs/reference/search/request/script-fields.asciidoc +++ b/docs/reference/search/request/script-fields.asciidoc @@ -37,9 +37,9 @@ Script fields can work on fields that are not stored (`my_field_name` in the above case), and allow to return custom values to be returned (the evaluated value of the script). -Script fields can also access the actual `_source` document indexed and -extract specific elements to be returned from it (can be an "object" -type). Here is an example: +Script fields can also access the actual `_source` document and +extract specific elements to be returned from it by using `params._source`. +Here is an example: [source,js] -------------------------------------------------- @@ -50,22 +50,24 @@ GET /_search }, "script_fields" : { "test1" : { - "script" : "_source.obj1.obj2" + "script" : "params['_source']['message']" } } } -------------------------------------------------- // CONSOLE +// TEST[setup:twitter] Note the `_source` keyword here to navigate the json-like model. It's important to understand the difference between -`doc['my_field'].value` and `_source.my_field`. The first, using the doc -keyword, will cause the terms for that field to be loaded to memory -(cached), which will result in faster execution, but more memory +`doc['my_field'].value` and `params['_source']['my_field']`. The first, +using the doc keyword, will cause the terms for that field to be loaded to +memory (cached), which will result in faster execution, but more memory consumption. Also, the `doc[...]` notation only allows for simple valued fields (can't return a json object from it) and make sense only on -non-analyzed or single term based fields. +non-analyzed or single term based fields. However, using `doc` is +still the recommended way to access values from the document, if at all +possible, because `_source` must be loaded and parsed every time it's used. +Using `_source` is very slow. -The `_source` on the other hand causes the source to be loaded, parsed, -and then only the relevant part of the json is returned. From 8e04561c0d9354955bc8c0d54a02786c60e756db Mon Sep 17 00:00:00 2001 From: Jack Conradson Date: Wed, 15 Mar 2017 17:29:31 -0700 Subject: [PATCH 05/16] Change params._source to params['_source'] in example. --- docs/reference/search/request/script-fields.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference/search/request/script-fields.asciidoc b/docs/reference/search/request/script-fields.asciidoc index b8132db7aadf1..c8886f779a5c6 100644 --- a/docs/reference/search/request/script-fields.asciidoc +++ b/docs/reference/search/request/script-fields.asciidoc @@ -38,7 +38,7 @@ the above case), and allow to return custom values to be returned (the evaluated value of the script). Script fields can also access the actual `_source` document and -extract specific elements to be returned from it by using `params._source`. +extract specific elements to be returned from it by using `params['_source']`. Here is an example: [source,js] From 8b6d521037b4c6109b59e03fb979d2c540897930 Mon Sep 17 00:00:00 2001 From: "Md.Abdulla-Al-Sun" Date: Thu, 16 Mar 2017 16:20:42 +0600 Subject: [PATCH 06/16] Remove Settings.settingsBuilder (#23575) In this repository, `Settings.builder` is used everywhere although it does exactly same as `Settings.settingsBuilder`. With the reference of the commit https://github.com/elastic/elasticsearch/commit/42526ac28e07da0055faafca1de6f8c5ec96cd85 , I think mistakenly this `Settings.settingsBuilder` remains in. --- docs/plugins/repository-azure.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/plugins/repository-azure.asciidoc b/docs/plugins/repository-azure.asciidoc index a1f8c6ea81d57..e6d8c682ba7a1 100644 --- a/docs/plugins/repository-azure.asciidoc +++ b/docs/plugins/repository-azure.asciidoc @@ -179,7 +179,7 @@ Example using Java: [source,java] ---- client.admin().cluster().preparePutRepository("my_backup_java1") - .setType("azure").setSettings(Settings.settingsBuilder() + .setType("azure").setSettings(Settings.builder() .put(Storage.CONTAINER, "backup-container") .put(Storage.CHUNK_SIZE, new ByteSizeValue(32, ByteSizeUnit.MB)) ).get(); From 6e9dfb334889258ff9205004ed8a851e1bc67d83 Mon Sep 17 00:00:00 2001 From: Robin Stocker Date: Fri, 17 Mar 2017 02:39:45 +1000 Subject: [PATCH 07/16] Docs: Specify that byte units use powers of 1024 (#23574) In SI units, "kilobyte" or "kB" would mean 1000 bytes, whereas "KiB" is used for 1024. Add a note in `api-conventions.asciidoc` to clarify the meaning in Elasticsearch. --- docs/reference/api-conventions.asciidoc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/reference/api-conventions.asciidoc b/docs/reference/api-conventions.asciidoc index 21bd539a4eaaf..7176f8858315d 100644 --- a/docs/reference/api-conventions.asciidoc +++ b/docs/reference/api-conventions.asciidoc @@ -506,8 +506,8 @@ the unit, like `2d` for 2 days. The supported units are: === Byte size units Whenever the byte size of data needs to be specified, eg when setting a buffer size -parameter, the value must specify the unit, like `10kb` for 10 kilobytes. The -supported units are: +parameter, the value must specify the unit, like `10kb` for 10 kilobytes. Note that +these units use powers of 1024, so `1kb` means 1024 bytes. The supported units are: [horizontal] `b`:: Bytes From 96a92da682f71f49764affeed5629b32d8f226d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Thu, 16 Mar 2017 21:36:18 +0100 Subject: [PATCH 08/16] CompletionSuggestionContext#toQuery() should also consider text if prefix/regex missing (#23451) In cases where the user specifies only the `text` option on the top level suggest element (either via REST or the java api), this gets transferred to the `text` property in the SuggestionSearchContext. CompletionSuggestionContext currently requires prefix or regex to be specified, otherwise errors. We should use the global `text` property as a fallback if neither prefix nor regex is provided. Closes to #23340 --- .../CompletionSuggestionContext.java | 29 +++++++++----- .../suggest/CompletionSuggestSearchIT.java | 38 ++++++++++++++++--- 2 files changed, 52 insertions(+), 15 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestionContext.java b/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestionContext.java index a4aeec8cb5833..b12b90de107ed 100644 --- a/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestionContext.java +++ b/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestionContext.java @@ -19,6 +19,7 @@ package org.elasticsearch.search.suggest.completion; import org.apache.lucene.search.suggest.document.CompletionQuery; +import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.unit.Fuzziness; import org.elasticsearch.index.mapper.CompletionFieldMapper; import org.elasticsearch.index.query.QueryShardContext; @@ -77,15 +78,7 @@ CompletionQuery toQuery() { CompletionFieldMapper.CompletionFieldType fieldType = getFieldType(); final CompletionQuery query; if (getPrefix() != null) { - if (fuzzyOptions != null) { - query = fieldType.fuzzyQuery(getPrefix().utf8ToString(), - Fuzziness.fromEdits(fuzzyOptions.getEditDistance()), - fuzzyOptions.getFuzzyPrefixLength(), fuzzyOptions.getFuzzyMinLength(), - fuzzyOptions.getMaxDeterminizedStates(), fuzzyOptions.isTranspositions(), - fuzzyOptions.isUnicodeAware()); - } else { - query = fieldType.prefixQuery(getPrefix()); - } + query = createCompletionQuery(getPrefix(), fieldType); } else if (getRegex() != null) { if (fuzzyOptions != null) { throw new IllegalArgumentException("can not use 'fuzzy' options with 'regex"); @@ -95,8 +88,10 @@ CompletionQuery toQuery() { } query = fieldType.regexpQuery(getRegex(), regexOptions.getFlagsValue(), regexOptions.getMaxDeterminizedStates()); + } else if (getText() != null) { + query = createCompletionQuery(getText(), fieldType); } else { - throw new IllegalArgumentException("'prefix' or 'regex' must be defined"); + throw new IllegalArgumentException("'prefix/text' or 'regex' must be defined"); } if (fieldType.hasContextMappings()) { ContextMappings contextMappings = fieldType.getContextMappings(); @@ -105,4 +100,18 @@ CompletionQuery toQuery() { return query; } + private CompletionQuery createCompletionQuery(BytesRef prefix, CompletionFieldMapper.CompletionFieldType fieldType) { + final CompletionQuery query; + if (fuzzyOptions != null) { + query = fieldType.fuzzyQuery(prefix.utf8ToString(), + Fuzziness.fromEdits(fuzzyOptions.getEditDistance()), + fuzzyOptions.getFuzzyPrefixLength(), fuzzyOptions.getFuzzyMinLength(), + fuzzyOptions.getMaxDeterminizedStates(), fuzzyOptions.isTranspositions(), + fuzzyOptions.isUnicodeAware()); + } else { + query = fieldType.prefixQuery(prefix); + } + return query; + } + } diff --git a/core/src/test/java/org/elasticsearch/search/suggest/CompletionSuggestSearchIT.java b/core/src/test/java/org/elasticsearch/search/suggest/CompletionSuggestSearchIT.java index 07502ff338379..5bd2bad31d134 100644 --- a/core/src/test/java/org/elasticsearch/search/suggest/CompletionSuggestSearchIT.java +++ b/core/src/test/java/org/elasticsearch/search/suggest/CompletionSuggestSearchIT.java @@ -68,12 +68,10 @@ import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAllSuccessful; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchHit; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.hasId; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.hasScore; import static org.hamcrest.Matchers.contains; -import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; @@ -116,6 +114,36 @@ public void testPrefix() throws Exception { assertSuggestions("foo", prefix, "suggestion10", "suggestion9", "suggestion8", "suggestion7", "suggestion6"); } + /** + * test that suggestion works if prefix is either provided via {@link CompletionSuggestionBuilder#text(String)} or + * {@link SuggestBuilder#setGlobalText(String)} + */ + public void testTextAndGlobalText() throws Exception { + final CompletionMappingBuilder mapping = new CompletionMappingBuilder(); + createIndexAndMapping(mapping); + int numDocs = 10; + List indexRequestBuilders = new ArrayList<>(); + for (int i = 1; i <= numDocs; i++) { + indexRequestBuilders.add(client().prepareIndex(INDEX, TYPE, "" + i).setSource(jsonBuilder().startObject().startObject(FIELD) + .field("input", "suggestion" + i).field("weight", i).endObject().endObject())); + } + indexRandom(true, indexRequestBuilders); + CompletionSuggestionBuilder noText = SuggestBuilders.completionSuggestion(FIELD); + SearchResponse searchResponse = client().prepareSearch(INDEX) + .suggest(new SuggestBuilder().addSuggestion("foo", noText).setGlobalText("sugg")).execute().actionGet(); + assertSuggestions(searchResponse, "foo", "suggestion10", "suggestion9", "suggestion8", "suggestion7", "suggestion6"); + + CompletionSuggestionBuilder withText = SuggestBuilders.completionSuggestion(FIELD).text("sugg"); + searchResponse = client().prepareSearch(INDEX) + .suggest(new SuggestBuilder().addSuggestion("foo", withText)).execute().actionGet(); + assertSuggestions(searchResponse, "foo", "suggestion10", "suggestion9", "suggestion8", "suggestion7", "suggestion6"); + + // test that suggestion text takes precedence over global text + searchResponse = client().prepareSearch(INDEX) + .suggest(new SuggestBuilder().addSuggestion("foo", withText).setGlobalText("bogus")).execute().actionGet(); + assertSuggestions(searchResponse, "foo", "suggestion10", "suggestion9", "suggestion8", "suggestion7", "suggestion6"); + } + public void testRegex() throws Exception { final CompletionMappingBuilder mapping = new CompletionMappingBuilder(); createIndexAndMapping(mapping); @@ -217,7 +245,7 @@ public void testSuggestDocument() throws Exception { for (CompletionSuggestion.Entry.Option option : options) { assertThat(option.getText().toString(), equalTo("suggestion" + id)); assertSearchHit(option.getHit(), hasId("" + id)); - assertSearchHit(option.getHit(), hasScore(((float) id))); + assertSearchHit(option.getHit(), hasScore((id))); assertNotNull(option.getHit().getSourceAsMap()); id--; } @@ -252,7 +280,7 @@ public void testSuggestDocumentNoSource() throws Exception { for (CompletionSuggestion.Entry.Option option : options) { assertThat(option.getText().toString(), equalTo("suggestion" + id)); assertSearchHit(option.getHit(), hasId("" + id)); - assertSearchHit(option.getHit(), hasScore(((float) id))); + assertSearchHit(option.getHit(), hasScore((id))); assertNull(option.getHit().getSourceAsMap()); id--; } @@ -289,7 +317,7 @@ public void testSuggestDocumentSourceFiltering() throws Exception { for (CompletionSuggestion.Entry.Option option : options) { assertThat(option.getText().toString(), equalTo("suggestion" + id)); assertSearchHit(option.getHit(), hasId("" + id)); - assertSearchHit(option.getHit(), hasScore(((float) id))); + assertSearchHit(option.getHit(), hasScore((id))); assertNotNull(option.getHit().getSourceAsMap()); Set sourceFields = option.getHit().getSourceAsMap().keySet(); assertThat(sourceFields, contains("a")); From b9ac69cbd838a8dc372092249d25ead4f415b038 Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Thu, 16 Mar 2017 22:02:15 -0400 Subject: [PATCH 09/16] Unmark reindex as experimental The reindex API is mature now, and we will work to maintain backwards compatibility in accordance with our backwards compatibility policy. This commit unmarks the reindex API as experimental. Relates #23621 --- docs/java-api/docs/update-by-query.asciidoc | 2 -- docs/reference/docs/delete-by-query.asciidoc | 2 -- docs/reference/docs/reindex.asciidoc | 2 -- 3 files changed, 6 deletions(-) diff --git a/docs/java-api/docs/update-by-query.asciidoc b/docs/java-api/docs/update-by-query.asciidoc index a94899668efa2..256ed32679437 100644 --- a/docs/java-api/docs/update-by-query.asciidoc +++ b/docs/java-api/docs/update-by-query.asciidoc @@ -1,8 +1,6 @@ [[docs-update-by-query]] == Update By Query API -experimental[The update-by-query API is new and should still be considered experimental. The API may change in ways that are not backwards compatible] - The simplest usage of `updateByQuery` updates each document in an index without changing the source. This usage enables <> or another online diff --git a/docs/reference/docs/delete-by-query.asciidoc b/docs/reference/docs/delete-by-query.asciidoc index db5804d5984c5..1e26aac6d612f 100644 --- a/docs/reference/docs/delete-by-query.asciidoc +++ b/docs/reference/docs/delete-by-query.asciidoc @@ -1,8 +1,6 @@ [[docs-delete-by-query]] == Delete By Query API -experimental[The delete-by-query API is new and should still be considered experimental. The API may change in ways that are not backwards compatible] - The simplest usage of `_delete_by_query` just performs a deletion on every document that match a query. Here is the API: diff --git a/docs/reference/docs/reindex.asciidoc b/docs/reference/docs/reindex.asciidoc index 0207a372fecbb..bd670fdf84b2f 100644 --- a/docs/reference/docs/reindex.asciidoc +++ b/docs/reference/docs/reindex.asciidoc @@ -1,8 +1,6 @@ [[docs-reindex]] == Reindex API -experimental[The reindex API is new and should still be considered experimental. The API may change in ways that are not backwards compatible] - IMPORTANT: Reindex does not attempt to set up the destination index. It does not copy the settings of the source index. You should set up the destination index prior to running a `_reindex` action, including setting up mappings, shard From e37cdab87f05eecd8ef62d880d15833f58502aaa Mon Sep 17 00:00:00 2001 From: Clinton Gormley Date: Thu, 16 Mar 2017 19:37:38 -0700 Subject: [PATCH 10/16] Update scripting.asciidoc Fixed bad asciidoc --- docs/reference/migration/migrate_6_0/scripting.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference/migration/migrate_6_0/scripting.asciidoc b/docs/reference/migration/migrate_6_0/scripting.asciidoc index 8a1eeaf639208..6740200922327 100644 --- a/docs/reference/migration/migrate_6_0/scripting.asciidoc +++ b/docs/reference/migration/migrate_6_0/scripting.asciidoc @@ -8,7 +8,7 @@ Use painless instead. ==== Date fields now return dates -`doc.some_date_field.value` now returns `ReadableDateTime`s instead of +`doc.some_date_field.value` now returns ++ReadableDateTime++s instead of milliseconds since epoch as a `long`. The same is true for `doc.some_date_field[some_number]`. Use `doc.some_date_field.value.millis` to fetch the milliseconds since epoch if you need it. From 90929f77ca860b542f46f612aa724d796aca700e Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Fri, 17 Mar 2017 01:05:48 -0400 Subject: [PATCH 11/16] Adapter action future should restore interrupts When a thread blocking on an adapter action future is interrupted, we throw an illegal state exception. This is documented, but it is rude to not restore the interrupt flag. This commit restores the interrupt flag in this situation, and adds a test. Relates #23618 --- .../action/support/AdapterActionFuture.java | 3 + .../support/AdapterActionFutureTests.java | 92 +++++++++++++++++++ 2 files changed, 95 insertions(+) create mode 100644 core/src/test/java/org/elasticsearch/action/support/AdapterActionFutureTests.java diff --git a/core/src/main/java/org/elasticsearch/action/support/AdapterActionFuture.java b/core/src/main/java/org/elasticsearch/action/support/AdapterActionFuture.java index b2167c3051bcb..4c7698e82e04d 100644 --- a/core/src/main/java/org/elasticsearch/action/support/AdapterActionFuture.java +++ b/core/src/main/java/org/elasticsearch/action/support/AdapterActionFuture.java @@ -38,6 +38,7 @@ public T actionGet() { try { return get(); } catch (InterruptedException e) { + Thread.currentThread().interrupt(); throw new IllegalStateException("Future got interrupted", e); } catch (ExecutionException e) { throw rethrowExecutionException(e); @@ -66,6 +67,7 @@ public T actionGet(long timeout, TimeUnit unit) { } catch (TimeoutException e) { throw new ElasticsearchTimeoutException(e); } catch (InterruptedException e) { + Thread.currentThread().interrupt(); throw new IllegalStateException("Future got interrupted", e); } catch (ExecutionException e) { throw rethrowExecutionException(e); @@ -100,4 +102,5 @@ public void onFailure(Exception e) { } protected abstract T convert(L listenerResponse); + } diff --git a/core/src/test/java/org/elasticsearch/action/support/AdapterActionFutureTests.java b/core/src/test/java/org/elasticsearch/action/support/AdapterActionFutureTests.java new file mode 100644 index 0000000000000..7ab9fa2dfe498 --- /dev/null +++ b/core/src/test/java/org/elasticsearch/action/support/AdapterActionFutureTests.java @@ -0,0 +1,92 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.support; + +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.test.ESTestCase; + +import java.util.Objects; +import java.util.concurrent.BrokenBarrierException; +import java.util.concurrent.CyclicBarrier; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +public class AdapterActionFutureTests extends ESTestCase { + + public void testInterruption() throws Exception { + final AdapterActionFuture adapter = + new AdapterActionFuture() { + @Override + protected String convert(final Integer listenerResponse) { + return Objects.toString(listenerResponse); + } + }; + + // test all possible methods that can be interrupted + final Runnable runnable = () -> { + final int method = randomIntBetween(0, 4); + switch (method) { + case 0: + adapter.actionGet(); + break; + case 1: + adapter.actionGet("30s"); + break; + case 2: + adapter.actionGet(30000); + break; + case 3: + adapter.actionGet(TimeValue.timeValueSeconds(30)); + break; + case 4: + adapter.actionGet(30, TimeUnit.SECONDS); + break; + default: + throw new AssertionError(method); + } + }; + + final CyclicBarrier barrier = new CyclicBarrier(2); + final Thread main = Thread.currentThread(); + final Thread thread = new Thread(() -> { + try { + barrier.await(); + } catch (final BrokenBarrierException | InterruptedException e) { + throw new RuntimeException(e); + } + main.interrupt(); + }); + thread.start(); + + barrier.await(); + + final AtomicBoolean interrupted = new AtomicBoolean(); + try { + runnable.run(); + } catch (final IllegalStateException e) { + interrupted.set(Thread.currentThread().isInterrupted()); + } + // we check this here instead of in the catch block to ensure that the catch block executed + assertTrue(interrupted.get()); + + thread.join(); + } + +} From 5bd14424e6b36541e174ca9e4ccb99e2bd5c45a7 Mon Sep 17 00:00:00 2001 From: Yannick Welsch Date: Fri, 17 Mar 2017 02:05:46 -0700 Subject: [PATCH 12/16] Fix third-party audit task for Gradle 3.4 (#23612) In Gradle 3.4, the buildSrc plugin seems to be packaged into a jar before it is accessed by the rest of the build and the signatures file for the third-party audit task cannot be accessed as getClass().getResource('/forbidden/third-party-audit.txt') then points to a file entry in a JAR, which cannot be loaded directly as a File object. This commit changes the third-party audit task to pass the content of the signatures file as a String instead. --- .../elasticsearch/gradle/precommit/ThirdPartyAuditTask.groovy | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/ThirdPartyAuditTask.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/ThirdPartyAuditTask.groovy index 018f9fde2f2c4..33ca6dccfa32e 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/ThirdPartyAuditTask.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/ThirdPartyAuditTask.groovy @@ -209,9 +209,11 @@ public class ThirdPartyAuditTask extends AntTask { try { ant.thirdPartyAudit(failOnUnsupportedJava: false, failOnMissingClasses: false, - signaturesFile: new File(getClass().getResource('/forbidden/third-party-audit.txt').toURI()), classpath: classpath.asPath) { fileset(dir: tmpDir) + signatures { + string(value: getClass().getResourceAsStream('/forbidden/third-party-audit.txt').getText('UTF-8')) + } } } catch (BuildException ignore) {} From 413bf0595600c4e961cf11a99ac9fc9172eba877 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Fri, 17 Mar 2017 14:07:18 +0100 Subject: [PATCH 13/16] Docs: Add comma to reverse nested agg snippet --- .../aggregations/bucket/reverse-nested-aggregation.asciidoc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/reference/aggregations/bucket/reverse-nested-aggregation.asciidoc b/docs/reference/aggregations/bucket/reverse-nested-aggregation.asciidoc index 9dba1f2adf000..b6074298e1c03 100644 --- a/docs/reference/aggregations/bucket/reverse-nested-aggregation.asciidoc +++ b/docs/reference/aggregations/bucket/reverse-nested-aggregation.asciidoc @@ -22,9 +22,9 @@ the issue documents as nested documents. The mapping could look like: "issue" : { "properties" : { - "tags" : { "type" : "text" } + "tags" : { "type" : "text" }, "comments" : { <1> - "type" : "nested" + "type" : "nested", "properties" : { "username" : { "type" : "keyword" }, "comment" : { "type" : "text" } From 89cbb0fed52c68aa3b4a1c70f176d5d401df276b Mon Sep 17 00:00:00 2001 From: Arnaud Venturi Date: Sat, 18 Mar 2017 00:56:31 +1000 Subject: [PATCH 14/16] Docs: Corrected path to elasticsearch-plugin (#23622) --- docs/plugins/plugin-script.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/plugins/plugin-script.asciidoc b/docs/plugins/plugin-script.asciidoc index 3234b6ae226f5..3a3616ac0e7ef 100644 --- a/docs/plugins/plugin-script.asciidoc +++ b/docs/plugins/plugin-script.asciidoc @@ -20,7 +20,7 @@ sudo bin/elasticsearch-plugin -h .Running as root ===================== If Elasticsearch was installed using the deb or rpm package then run -`/usr/share/elasticsearch-plugin` as `root` so it can write to the appropriate files on disk. +`/usr/share/elasticsearch/bin/elasticsearch-plugin` as `root` so it can write to the appropriate files on disk. Otherwise run `bin/elasticsearch-plugin` as the user that owns all of the Elasticsearch files. ===================== From e8b2f9afd4d38b1e279e15d57c78ac033f9569f9 Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Fri, 17 Mar 2017 11:19:58 -0400 Subject: [PATCH 15/16] Migrate to max line length of 100 This commit moves the checkstyle rule of max line length from 140 characters to 100 characters. We whitelist all existing violations and will address them in follow-ups. Relates #23623 --- buildSrc/src/main/resources/checkstyle.xml | 2 +- .../resources/checkstyle_suppressions.xml | 3505 ++++++++++++++++- 2 files changed, 3348 insertions(+), 159 deletions(-) diff --git a/buildSrc/src/main/resources/checkstyle.xml b/buildSrc/src/main/resources/checkstyle.xml index 891a85d50a930..85b55a71cf82a 100644 --- a/buildSrc/src/main/resources/checkstyle.xml +++ b/buildSrc/src/main/resources/checkstyle.xml @@ -22,7 +22,7 @@ suppress the check there but enforce it everywhere else. This prevents the list from getting longer even if it is unfair. --> - + diff --git a/buildSrc/src/main/resources/checkstyle_suppressions.xml b/buildSrc/src/main/resources/checkstyle_suppressions.xml index 85a658df5ac3f..659d31186f126 100644 --- a/buildSrc/src/main/resources/checkstyle_suppressions.xml +++ b/buildSrc/src/main/resources/checkstyle_suppressions.xml @@ -10,547 +10,2419 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - + + + + + - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - + + + + + + + - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + - - + + + + + + + - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + + + + + + + + + + + + + + + + + + + + + + - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + - + + + + + + + + + + + + + + + + + + + + + - + + + + + + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - + + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + + + + + + + + + + + + + + + + + + + + + + + + + - + + + + + + + - + + + + + + + + + + + + + + + + + + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + + + + - + + + + + + + + + + - + + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - + + + + + + + + + + + + + + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -562,37 +2434,69 @@ - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -602,6 +2506,8 @@ + + @@ -610,159 +2516,443 @@ + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + + + + + + + + + + - + + + + + + - + + + + + + + + + + + + + + + + + + + + + + + + + - - - - + + + + + + + + + + + + + - + + + + + + - - - - + + + + + + + + + + - - - - - + + - - - - - + + + + + + + + + + + + + + + - - + + + + + + + - + - - - - - + + + + + + + + + - + + + + + + + + + - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + + + + + @@ -771,189 +2961,1188 @@ + + + + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + + + + + - - + + + + + + + + + + + + + + + + + + + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - + + - + + + + + + + + + + + + + + + + + + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + + + + + + + - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + + + + + + + + + + + + - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + + + + + + - + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From c462d7d486bf23a507acfbc80445617021b07e53 Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Fri, 17 Mar 2017 12:46:23 -0400 Subject: [PATCH 16/16] Clear the interrupt flag before joining This commit changes the method for checking the interrupt status of a thread that is intentionally interrupted during AdapterActionFutureTests#testInteruption. Namely, we want to check and clear the interrupt status before joining on the interrupting thread. If we do not clear the status, when we lose a race where the interrupting thread is not yet finished, an interrupted exception will be thrown when we try to join on it. Clearing the interrupted status on the main thread addresses this issue. --- .../action/support/AdapterActionFutureTests.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/core/src/test/java/org/elasticsearch/action/support/AdapterActionFutureTests.java b/core/src/test/java/org/elasticsearch/action/support/AdapterActionFutureTests.java index 7ab9fa2dfe498..a7405ddae8cce 100644 --- a/core/src/test/java/org/elasticsearch/action/support/AdapterActionFutureTests.java +++ b/core/src/test/java/org/elasticsearch/action/support/AdapterActionFutureTests.java @@ -75,13 +75,14 @@ protected String convert(final Integer listenerResponse) { }); thread.start(); + final AtomicBoolean interrupted = new AtomicBoolean(); + barrier.await(); - final AtomicBoolean interrupted = new AtomicBoolean(); try { runnable.run(); } catch (final IllegalStateException e) { - interrupted.set(Thread.currentThread().isInterrupted()); + interrupted.set(Thread.interrupted()); } // we check this here instead of in the catch block to ensure that the catch block executed assertTrue(interrupted.get());