diff --git a/build.gradle b/build.gradle index 6989646d72..eba53b88b6 100644 --- a/build.gradle +++ b/build.gradle @@ -34,6 +34,7 @@ plugins { id 'nebula.ospackage' version "5.3.0" id 'java-library' id 'checkstyle' + id "io.freefair.lombok" version "4.1.2" } /* @@ -99,6 +100,7 @@ esplugin { // TODO: fix compiler warnings compileJava.options.warnings = false compileJava { + options.compilerArgs.addAll(["-processor", 'lombok.launch.AnnotationProcessorHider$AnnotationProcessor']) doFirst { // TODO: do not fail build on warnings, need to fix all compiler warnings options.compilerArgs.remove('-Werror') diff --git a/docs/user/admin/settings.rst b/docs/user/admin/settings.rst index dd4b9d179b..af39166dc4 100644 --- a/docs/user/admin/settings.rst +++ b/docs/user/admin/settings.rst @@ -39,7 +39,7 @@ SQL query:: >> curl -H 'Content-Type: application/json' -X PUT localhost:9200/_cluster/settings -d '{ "transient" : { - "opendistro.sql.enabled" : false + "opendistro.sql.enabled" : "false" } }' @@ -101,7 +101,7 @@ SQL query:: >> curl -H 'Content-Type: application/json' -X PUT localhost:9200/_cluster/settings -d '{ "transient" : { - "opendistro.sql.query.slowlog" : 10 + "opendistro.sql.query.slowlog" : "10" } }' @@ -143,7 +143,7 @@ SQL query:: >> curl -H 'Content-Type: application/json' -X PUT localhost:9200/_cluster/settings -d '{ "transient" : { - "opendistro.sql.query.analysis.enabled" : false + "opendistro.sql.query.analysis.enabled" : "false" } }' @@ -187,7 +187,7 @@ SQL query:: >> curl -H 'Content-Type: application/json' -X PUT localhost:9200/_cluster/settings -d '{ "transient" : { - "opendistro.sql.query.analysis.semantic.suggestion" : true + "opendistro.sql.query.analysis.semantic.suggestion" : "true" } }' @@ -255,7 +255,7 @@ SQL query:: >> curl -H 'Content-Type: application/json' -X PUT localhost:9200/_cluster/settings -d '{ "transient" : { - "opendistro.sql.query.analysis.semantic.threshold" : 50 + "opendistro.sql.query.analysis.semantic.threshold" : "50" } }' @@ -279,3 +279,108 @@ Result set:: } } +opendistro.sql.query.response.format +==================================== + +Description +----------- + +User can set default response format of the query. The supported format includes: jdbc,json,csv,raw,table. + +1. The default value is jdbc. +2. This setting is node scope. +3. This setting can be updated dynamically. + + +Example 1 +--------- + +You can update the setting with a new value like this. + +SQL query:: + + >> curl -H 'Content-Type: application/json' -X PUT localhost:9200/_cluster/settings -d '{ + "transient" : { + "opendistro.sql.query.response.format" : "json" + } + }' + +Result set:: + + { + "acknowledged" : true, + "persistent" : { }, + "transient" : { + "opendistro" : { + "sql" : { + "query" : { + "response" : { + "format" : "json" + } + } + } + } + } + } + +Example 2 +--------- + +Query result after the setting updated is like: + +SQL query:: + + >> curl -H 'Content-Type: application/json' -X POST localhost:9200/_opendistro/_sql -d '{ + "query" : "SELECT firstname, lastname, age FROM accounts ORDER BY age LIMIT 2" + }' + +Result set:: + + { + "_shards" : { + "total" : 5, + "failed" : 0, + "successful" : 5, + "skipped" : 0 + }, + "hits" : { + "hits" : [ + { + "_index" : "accounts", + "_type" : "account", + "_source" : { + "firstname" : "Nanette", + "age" : 28, + "lastname" : "Bates" + }, + "_id" : "13", + "sort" : [ + 28 + ], + "_score" : null + }, + { + "_index" : "accounts", + "_type" : "account", + "_source" : { + "firstname" : "Amber", + "age" : 32, + "lastname" : "Duke" + }, + "_id" : "1", + "sort" : [ + 32 + ], + "_score" : null + } + ], + "total" : { + "value" : 4, + "relation" : "eq" + }, + "max_score" : null + }, + "took" : 100, + "timed_out" : false + } + diff --git a/docs/user/interfaces/protocol.rst b/docs/user/interfaces/protocol.rst index 6b2b83e9c2..e9d808248f 100644 --- a/docs/user/interfaces/protocol.rst +++ b/docs/user/interfaces/protocol.rst @@ -133,80 +133,13 @@ Explain:: } } -Elasticsearch DSL -================= - -Description ------------ - -By default the plugin returns original response from Elasticsearch in JSON. Because this is the native response from Elasticsearch, extra efforts are needed to parse and interpret it. - -Example -------- - -SQL query:: - - >> curl -H 'Content-Type: application/json' -X POST localhost:9200/_opendistro/_sql -d '{ - "query" : "SELECT firstname, lastname, age FROM accounts ORDER BY age LIMIT 2" - }' - -Result set:: - - { - "_shards" : { - "total" : 5, - "failed" : 0, - "successful" : 5, - "skipped" : 0 - }, - "hits" : { - "hits" : [ - { - "_index" : "accounts", - "_type" : "account", - "_source" : { - "firstname" : "Nanette", - "age" : 28, - "lastname" : "Bates" - }, - "_id" : "13", - "sort" : [ - 28 - ], - "_score" : null - }, - { - "_index" : "accounts", - "_type" : "account", - "_source" : { - "firstname" : "Amber", - "age" : 32, - "lastname" : "Duke" - }, - "_id" : "1", - "sort" : [ - 32 - ], - "_score" : null - } - ], - "total" : { - "value" : 4, - "relation" : "eq" - }, - "max_score" : null - }, - "took" : 100, - "timed_out" : false - } - JDBC Format =========== Description ----------- -JDBC format is provided for JDBC driver and client side that needs both schema and result set well formatted. +By default the plugin return JDBC format. JDBC format is provided for JDBC driver and client side that needs both schema and result set well formatted. Example 1 --------- @@ -215,7 +148,7 @@ Here is an example for normal response. The `schema` includes field name and its SQL query:: - >> curl -H 'Content-Type: application/json' -X POST localhost:9200/_opendistro/_sql?format=jdbc -d '{ + >> curl -H 'Content-Type: application/json' -X POST localhost:9200/_opendistro/_sql -d '{ "query" : "SELECT firstname, lastname, age FROM accounts ORDER BY age LIMIT 2" }' @@ -275,6 +208,73 @@ Result set:: "status" : 400 } +Elasticsearch DSL +================= + +Description +----------- + +The plugin returns original response from Elasticsearch in JSON. Because this is the native response from Elasticsearch, extra efforts are needed to parse and interpret it. + +Example +------- + +SQL query:: + + >> curl -H 'Content-Type: application/json' -X POST localhost:9200/_opendistro/_sql?format=json -d '{ + "query" : "SELECT firstname, lastname, age FROM accounts ORDER BY age LIMIT 2" + }' + +Result set:: + + { + "_shards" : { + "total" : 5, + "failed" : 0, + "successful" : 5, + "skipped" : 0 + }, + "hits" : { + "hits" : [ + { + "_index" : "accounts", + "_type" : "account", + "_source" : { + "firstname" : "Nanette", + "age" : 28, + "lastname" : "Bates" + }, + "_id" : "13", + "sort" : [ + 28 + ], + "_score" : null + }, + { + "_index" : "accounts", + "_type" : "account", + "_source" : { + "firstname" : "Amber", + "age" : 32, + "lastname" : "Duke" + }, + "_id" : "1", + "sort" : [ + 32 + ], + "_score" : null + } + ], + "total" : { + "value" : 4, + "relation" : "eq" + }, + "max_score" : null + }, + "took" : 100, + "timed_out" : false + } + CSV Format ========== diff --git a/lombok.config b/lombok.config new file mode 100644 index 0000000000..189c0bef98 --- /dev/null +++ b/lombok.config @@ -0,0 +1,3 @@ +# This file is generated by the 'io.freefair.lombok' Gradle plugin +config.stopBubbling = true +lombok.addLombokGeneratedAnnotation = true diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/ActionRequestRestExecutorFactory.java b/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/ActionRequestRestExecutorFactory.java index 13a44dfc44..63999fe891 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/ActionRequestRestExecutorFactory.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/ActionRequestRestExecutorFactory.java @@ -21,13 +21,10 @@ import com.amazon.opendistroforelasticsearch.sql.query.join.ESJoinQueryAction; import com.amazon.opendistroforelasticsearch.sql.query.multi.MultiQueryAction; -import java.util.stream.Stream; - /** * Created by Eliran on 26/12/2015. */ public class ActionRequestRestExecutorFactory { - /** * Create executor based on the format and wrap with AsyncRestExecutor * to async blocking execute() call if necessary. @@ -36,23 +33,21 @@ public class ActionRequestRestExecutorFactory { * @param queryAction query action * @return executor */ - public static RestExecutor createExecutor(String format, QueryAction queryAction) { - if (format == null || format.equals("")) { - return new AsyncRestExecutor( - new ElasticDefaultRestExecutor(queryAction), - action -> isJoin(action) || isUnionMinus(action) - ); - } - - if (format.equalsIgnoreCase("csv")) { - return new AsyncRestExecutor(new CSVResultRestExecutor()); + public static RestExecutor createExecutor(Format format, QueryAction queryAction) { + switch (format) { + case CSV: + return new AsyncRestExecutor(new CSVResultRestExecutor()); + case JSON: + return new AsyncRestExecutor( + new ElasticDefaultRestExecutor(queryAction), + action -> isJoin(action) || isUnionMinus(action) + ); + case JDBC: + case RAW: + case TABLE: + default: + return new AsyncRestExecutor(new PrettyFormatRestExecutor(format.getFormatName())); } - - if (Stream.of("jdbc", "table", "raw").anyMatch(format::equalsIgnoreCase)) { - return new AsyncRestExecutor(new PrettyFormatRestExecutor(format)); - } - - throw new IllegalArgumentException("Failed to create executor due to unknown response format: " + format); } private static boolean isJoin(QueryAction queryAction) { diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/Format.java b/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/Format.java new file mode 100644 index 0000000000..ab4ac5a56f --- /dev/null +++ b/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/Format.java @@ -0,0 +1,48 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.executor; + +import com.google.common.collect.ImmutableMap; +import lombok.Getter; +import lombok.RequiredArgsConstructor; + +import java.util.Map; +import java.util.Optional; + +@RequiredArgsConstructor +public enum Format { + JDBC("jdbc"), + JSON("json"), + CSV("csv"), + RAW("raw"), + TABLE("table"); + + @Getter + private final String formatName; + + private static final Map ALL_FORMATS; + static { + ImmutableMap.Builder builder = new ImmutableMap.Builder<>(); + for (Format format : Format.values()) { + builder.put(format.formatName, format); + } + ALL_FORMATS = builder.build(); + } + + public static Optional of(String formatName) { + return Optional.ofNullable(ALL_FORMATS.getOrDefault(formatName, null)); + } +} diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/plugin/RestSqlAction.java b/src/main/java/com/amazon/opendistroforelasticsearch/sql/plugin/RestSqlAction.java index af781e2421..f5e823944f 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/plugin/RestSqlAction.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/sql/plugin/RestSqlAction.java @@ -25,13 +25,14 @@ import com.amazon.opendistroforelasticsearch.sql.executor.ActionRequestRestExecutorFactory; import com.amazon.opendistroforelasticsearch.sql.executor.RestExecutor; import com.amazon.opendistroforelasticsearch.sql.executor.format.ErrorMessage; -import com.amazon.opendistroforelasticsearch.sql.utils.JsonPrettyFormatter; import com.amazon.opendistroforelasticsearch.sql.metrics.MetricName; import com.amazon.opendistroforelasticsearch.sql.metrics.Metrics; import com.amazon.opendistroforelasticsearch.sql.query.QueryAction; import com.amazon.opendistroforelasticsearch.sql.request.SqlRequest; import com.amazon.opendistroforelasticsearch.sql.request.SqlRequestFactory; +import com.amazon.opendistroforelasticsearch.sql.request.SqlRequestParam; import com.amazon.opendistroforelasticsearch.sql.rewriter.matchtoterm.VerificationException; +import com.amazon.opendistroforelasticsearch.sql.utils.JsonPrettyFormatter; import com.amazon.opendistroforelasticsearch.sql.utils.LogUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -151,15 +152,15 @@ private void executeSqlRequest(final RestRequest request, final QueryAction quer if (isExplainRequest(request)) { final String jsonExplanation = queryAction.explain().explain(); String result; - if (params.containsKey("pretty") - && ("".equals(params.get("pretty")) || "true".equals(params.get("pretty")))) { + if (SqlRequestParam.isPrettyFormat(params)) { result = JsonPrettyFormatter.format(jsonExplanation); } else { result = jsonExplanation; } channel.sendResponse(new BytesRestResponse(OK, "application/json; charset=UTF-8", result)); } else { - RestExecutor restExecutor = ActionRequestRestExecutorFactory.createExecutor(params.get("format"), + RestExecutor restExecutor = ActionRequestRestExecutorFactory.createExecutor( + SqlRequestParam.getFormat(params), queryAction); //doing this hack because elasticsearch throws exception for un-consumed props Map additionalParams = new HashMap<>(); diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/plugin/SqlSettings.java b/src/main/java/com/amazon/opendistroforelasticsearch/sql/plugin/SqlSettings.java index 0cd53cc5d7..a3e8700538 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/plugin/SqlSettings.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/sql/plugin/SqlSettings.java @@ -15,6 +15,7 @@ package com.amazon.opendistroforelasticsearch.sql.plugin; +import com.amazon.opendistroforelasticsearch.sql.executor.Format; import org.elasticsearch.common.settings.Setting; import java.util.ArrayList; @@ -38,6 +39,7 @@ public class SqlSettings { */ public static final String SQL_ENABLED = "opendistro.sql.enabled"; public static final String QUERY_SLOWLOG = "opendistro.sql.query.slowlog"; + public static final String QUERY_RESPONSE_FORMAT = "opendistro.sql.query.response.format"; public static final String QUERY_ANALYSIS_ENABLED = "opendistro.sql.query.analysis.enabled"; public static final String QUERY_ANALYSIS_SEMANTIC_SUGGESTION = "opendistro.sql.query.analysis.semantic.suggestion"; public static final String QUERY_ANALYSIS_SEMANTIC_THRESHOLD = "opendistro.sql.query.analysis.semantic.threshold"; @@ -50,6 +52,8 @@ public SqlSettings() { Map> settings = new HashMap<>(); settings.put(SQL_ENABLED, Setting.boolSetting(SQL_ENABLED, true, NodeScope, Dynamic)); settings.put(QUERY_SLOWLOG, Setting.intSetting(QUERY_SLOWLOG, 2, NodeScope, Dynamic)); + settings.put(QUERY_RESPONSE_FORMAT, Setting.simpleString(QUERY_RESPONSE_FORMAT, Format.JDBC.getFormatName(), + NodeScope, Dynamic)); // Settings for new ANTLR query analyzer settings.put(QUERY_ANALYSIS_ENABLED, Setting.boolSetting( diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/request/SqlRequestParam.java b/src/main/java/com/amazon/opendistroforelasticsearch/sql/request/SqlRequestParam.java new file mode 100644 index 0000000000..2367333d11 --- /dev/null +++ b/src/main/java/com/amazon/opendistroforelasticsearch/sql/request/SqlRequestParam.java @@ -0,0 +1,65 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.request; + +import com.amazon.opendistroforelasticsearch.sql.esdomain.LocalClusterState; +import com.amazon.opendistroforelasticsearch.sql.executor.Format; + +import java.util.Map; +import java.util.Optional; + +import static com.amazon.opendistroforelasticsearch.sql.plugin.SqlSettings.QUERY_RESPONSE_FORMAT; + +/** + * Utils class for parse the request params. + */ +public class SqlRequestParam { + public static final String QUERY_PARAMS_FORMAT = "format"; + public static final String QUERY_PARAMS_PRETTY = "pretty"; + + /** + * Parse the pretty params to decide whether the response should be pretty formatted. + * @param requestParams request params. + * @return return true if the response required pretty format, otherwise return false. + */ + public static boolean isPrettyFormat(Map requestParams) { + return requestParams.containsKey(QUERY_PARAMS_PRETTY) + && ("".equals(requestParams.get(QUERY_PARAMS_PRETTY)) + || "true".equals(requestParams.get(QUERY_PARAMS_PRETTY))); + } + + /** + * Parse the request params and return the {@link Format} of the response + * @param requestParams request params + * @return The response Format. + */ + public static Format getFormat(Map requestParams) { + String formatName; + if (requestParams.containsKey(QUERY_PARAMS_FORMAT)) { + formatName = requestParams.get(QUERY_PARAMS_FORMAT).toLowerCase(); + } else { + LocalClusterState clusterState = LocalClusterState.state(); + formatName = clusterState.getSettingValue(QUERY_RESPONSE_FORMAT); + } + Optional optionalFormat = Format.of(formatName); + if (optionalFormat.isPresent()) { + return optionalFormat.get(); + } else { + throw new IllegalArgumentException("Failed to create executor due to unknown response format: " + + formatName); + } + } +} diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/admin/PluginSettingIT.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/admin/PluginSettingIT.java index 8097ad4793..f43e29a9b3 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/admin/PluginSettingIT.java +++ b/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/admin/PluginSettingIT.java @@ -20,11 +20,14 @@ import com.amazon.opendistroforelasticsearch.sql.doctest.core.annotation.Section; import com.amazon.opendistroforelasticsearch.sql.doctest.core.builder.Example; import com.amazon.opendistroforelasticsearch.sql.doctest.core.builder.ListItems; +import com.amazon.opendistroforelasticsearch.sql.executor.Format; import com.amazon.opendistroforelasticsearch.sql.plugin.SqlSettings; import com.amazon.opendistroforelasticsearch.sql.utils.StringUtils; import org.elasticsearch.common.settings.Setting; +import java.util.Arrays; import java.util.EnumSet; +import java.util.stream.Collectors; import static com.amazon.opendistroforelasticsearch.sql.doctest.core.request.SqlRequestFormat.CURL_REQUEST; import static com.amazon.opendistroforelasticsearch.sql.doctest.core.request.SqlRequestFormat.IGNORE_REQUEST; @@ -33,6 +36,7 @@ import static com.amazon.opendistroforelasticsearch.sql.plugin.SqlSettings.QUERY_ANALYSIS_ENABLED; import static com.amazon.opendistroforelasticsearch.sql.plugin.SqlSettings.QUERY_ANALYSIS_SEMANTIC_SUGGESTION; import static com.amazon.opendistroforelasticsearch.sql.plugin.SqlSettings.QUERY_ANALYSIS_SEMANTIC_THRESHOLD; +import static com.amazon.opendistroforelasticsearch.sql.plugin.SqlSettings.QUERY_RESPONSE_FORMAT; import static com.amazon.opendistroforelasticsearch.sql.plugin.SqlSettings.QUERY_SLOWLOG; import static com.amazon.opendistroforelasticsearch.sql.plugin.SqlSettings.SQL_ENABLED; import static org.elasticsearch.common.settings.Setting.Property; @@ -99,6 +103,19 @@ public void semanticAnalysisThresholdSetting() { ); } + @Section(6) + public void responseFormatSetting() { + docSetting( + QUERY_RESPONSE_FORMAT, + String.format("User can set default response format of the query. " + + "The supported format includes: %s.", Arrays.stream(Format.values()) + .map(Format::getFormatName) + .collect(Collectors.joining(","))), + Format.JSON.getFormatName(), + "SELECT firstname, lastname, age FROM accounts ORDER BY age LIMIT 2" + ); + } + /** * Generate content for sample queries with setting changed to new value. * Finally setting will be reverted to avoid potential impact on other test cases. diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/builder/DocBuilder.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/builder/DocBuilder.java index 1d0a9db543..e4a2686416 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/builder/DocBuilder.java +++ b/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/core/builder/DocBuilder.java @@ -149,7 +149,9 @@ default Requests get(String sql) { } default Requests put(String name, Object value) { - String setting = StringUtils.format("\"%s\": {\"%s\": %s}", "transient", name, value); + String setting = value == null ? + StringUtils.format("\"%s\": {\"%s\": null}", "transient", name) : + StringUtils.format("\"%s\": {\"%s\": \"%s\"}", "transient", name, value); return new Requests( restClient(), new SqlRequest("PUT", "/_cluster/settings", new Body(setting).toString()), diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/interfaces/ProtocolIT.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/interfaces/ProtocolIT.java index 0469828bd8..30f3b82b5a 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/interfaces/ProtocolIT.java +++ b/src/test/java/com/amazon/opendistroforelasticsearch/sql/doctest/interfaces/ProtocolIT.java @@ -64,28 +64,11 @@ public void requestFormat() { } @Section(2) - public void originalDSLResponse() { - section( - title("Elasticsearch DSL"), - description( - "By default the plugin returns original response from Elasticsearch in JSON. Because this is", - "the native response from Elasticsearch, extra efforts are needed to parse and interpret it." - ), - example( - description(), - post("SELECT firstname, lastname, age FROM accounts ORDER BY age LIMIT 2"), - queryFormat(CURL_REQUEST, PRETTY_JSON_RESPONSE), - explainFormat(IGNORE_REQUEST, IGNORE_RESPONSE) - ) - ); - } - - @Section(3) public void responseInJDBCFormat() { section( title("JDBC Format"), description( - "JDBC format is provided for JDBC driver and client side that needs both schema and", + "By default the plugin return JDBC format. JDBC format is provided for JDBC driver and client side that needs both schema and", "result set well formatted." ), example( @@ -93,7 +76,7 @@ public void responseInJDBCFormat() { "Here is an example for normal response. The `schema` includes field name and its type", "and `datarows` includes the result set." ), - post("SELECT firstname, lastname, age FROM accounts ORDER BY age LIMIT 2", params("format=jdbc")), + post("SELECT firstname, lastname, age FROM accounts ORDER BY age LIMIT 2"), queryFormat(CURL_REQUEST, PRETTY_JSON_RESPONSE), explainFormat(IGNORE_REQUEST, IGNORE_RESPONSE) ), @@ -106,6 +89,23 @@ public void responseInJDBCFormat() { ); } + @Section(3) + public void originalDSLResponse() { + section( + title("Elasticsearch DSL"), + description( + "The plugin returns original response from Elasticsearch in JSON. Because this is", + "the native response from Elasticsearch, extra efforts are needed to parse and interpret it." + ), + example( + description(), + post("SELECT firstname, lastname, age FROM accounts ORDER BY age LIMIT 2", params("format=json")), + queryFormat(CURL_REQUEST, PRETTY_JSON_RESPONSE), + explainFormat(IGNORE_REQUEST, IGNORE_RESPONSE) + ) + ); + } + @Section(4) public void responseInCSVFormat() { section( diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/SQLIntegTestCase.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/SQLIntegTestCase.java index b91ccac8dd..0dcefad83c 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/SQLIntegTestCase.java +++ b/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/SQLIntegTestCase.java @@ -90,8 +90,8 @@ protected void loadIndex(Index index) throws Exception { } protected Request getSqlRequest(String request, boolean explain) { - - Request sqlRequest = new Request("POST", explain ? EXPLAIN_API_ENDPOINT : QUERY_API_ENDPOINT); + String queryEndpoint = String.format("%s?format=%s", QUERY_API_ENDPOINT, "json"); + Request sqlRequest = new Request("POST", explain ? EXPLAIN_API_ENDPOINT : queryEndpoint); sqlRequest.setJsonEntity(request); RequestOptions.Builder restOptionsBuilder = RequestOptions.DEFAULT.toBuilder(); restOptionsBuilder.addHeader("Content-Type", "application/json"); @@ -131,8 +131,8 @@ protected Request buildGetEndpointRequest(final String sqlQuery) { Assert.fail(utf8CharsetName + " not available"); } - final String requestUrl = String.format(Locale.ROOT, "%s?sql=%s", - QUERY_API_ENDPOINT, urlEncodedQuery); + final String requestUrl = String.format(Locale.ROOT, "%s?sql=%s&format=%s", QUERY_API_ENDPOINT, + urlEncodedQuery, "json"); return new Request("GET", requestUrl); } diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/FormatTest.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/FormatTest.java new file mode 100644 index 0000000000..d9ba2b666a --- /dev/null +++ b/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/FormatTest.java @@ -0,0 +1,40 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.unittest; + +import com.amazon.opendistroforelasticsearch.sql.executor.Format; +import org.junit.Test; + +import java.util.Optional; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +public class FormatTest { + + @Test + public void ofJdbcShouldReturnJDBCFormat() { + Optional format = Format.of(Format.JDBC.getFormatName()); + assertTrue(format.isPresent()); + assertEquals(Format.JDBC, format.get()); + } + + @Test + public void ofUnknownFormatShouldReturnEmpty() { + assertFalse(Format.of("xml").isPresent()); + } +} \ No newline at end of file diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/SqlRequestParamTest.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/SqlRequestParamTest.java new file mode 100644 index 0000000000..85845cd0cb --- /dev/null +++ b/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/SqlRequestParamTest.java @@ -0,0 +1,95 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.unittest; + +import com.amazon.opendistroforelasticsearch.sql.esdomain.LocalClusterState; +import com.amazon.opendistroforelasticsearch.sql.executor.Format; +import com.amazon.opendistroforelasticsearch.sql.plugin.SqlSettings; +import com.amazon.opendistroforelasticsearch.sql.request.SqlRequestParam; +import com.google.common.collect.ImmutableMap; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +import static com.amazon.opendistroforelasticsearch.sql.request.SqlRequestParam.QUERY_PARAMS_FORMAT; +import static com.amazon.opendistroforelasticsearch.sql.request.SqlRequestParam.QUERY_PARAMS_PRETTY; +import static java.util.Collections.emptyList; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.spy; + +public class SqlRequestParamTest { + @Rule + public ExpectedException exceptionRule = ExpectedException.none(); + + @Before + public void setup() { + SqlSettings settings = spy(new SqlSettings()); + // Force return empty list to avoid ClusterSettings be invoked which is a final class and hard to mock. + // In this case, default value in Setting will be returned all the time. + doReturn(emptyList()).when(settings).getSettings(); + LocalClusterState.state().setSqlSettings(settings); + } + + @Test + public void shouldReturnTrueIfPrettyParamsIsTrue() { + assertTrue(SqlRequestParam.isPrettyFormat(ImmutableMap.of(QUERY_PARAMS_PRETTY, "true"))); + } + + @Test + public void shouldReturnTrueIfPrettyParamsIsEmpty() { + assertTrue(SqlRequestParam.isPrettyFormat(ImmutableMap.of(QUERY_PARAMS_PRETTY, ""))); + } + + @Test + public void shouldReturnFalseIfNoPrettyParams() { + assertFalse(SqlRequestParam.isPrettyFormat(ImmutableMap.of())); + } + + @Test + public void shouldReturnFalseIfPrettyParamsIsUnknownValue() { + assertFalse(SqlRequestParam.isPrettyFormat(ImmutableMap.of(QUERY_PARAMS_PRETTY, "unknown"))); + } + + @Test + public void shouldReturnJSONIfFormatParamsIsJSON() { + assertEquals(Format.JSON, SqlRequestParam.getFormat(ImmutableMap.of(QUERY_PARAMS_FORMAT, "json"))); + } + + @Test + public void shouldReturnDefaultFormatIfNoFormatParams() { + assertEquals(Format.JDBC, SqlRequestParam.getFormat(ImmutableMap.of())); + } + + @Test + public void shouldThrowExceptionIfFormatParamsIsEmpty() { + exceptionRule.expect(IllegalArgumentException.class); + exceptionRule.expectMessage("Failed to create executor due to unknown response format: "); + + assertEquals(Format.JDBC, SqlRequestParam.getFormat(ImmutableMap.of(QUERY_PARAMS_FORMAT, ""))); + } + + @Test + public void shouldThrowExceptionIfFormatParamsIsNotSupported() { + exceptionRule.expect(IllegalArgumentException.class); + exceptionRule.expectMessage("Failed to create executor due to unknown response format: xml"); + + SqlRequestParam.getFormat(ImmutableMap.of(QUERY_PARAMS_FORMAT, "xml")); + } +} \ No newline at end of file