Address comments

docs/reference/sql/language/data-types.asciidoc

@@ -49,15 +49,15 @@ As one can see, all of {es} <<mapping-types, data types>> are mapped to the data
 name in {es-sql}, with the exception of **date** data type which is mapped to **datetime** in {es-sql}.
 This is to avoid confusion with the ANSI SQL **DATE** (date only) type, which is also supported by {es-sql}
 in queries (with the use of <<sql-functions-type-conversion-cast>>/<<sql-functions-type-conversion-convert>>),
-but doesn't correspond to an actual mapping in {es} (see the <<es-sql-extra-types, `table`>> below).
+but doesn't correspond to an actual mapping in {es} (see the <<es-sql-only-types, `table`>> below).
 
 Obviously, not all types in {es} have an equivalent in SQL and vice-versa hence why, {es-sql}
 uses the data type _particularities_ of the former over the latter as ultimately {es} is the backing store.
 
 In addition to the types above, {es-sql} also supports at _runtime_ SQL-specific types that do not have an equivalent in {es}.
 Such types cannot be loaded from {es} (as it does not know about them) however can be used inside {es-sql} in queries or their results.
 
-[[es-sql-extra-types]]
+[[es-sql-only-types]]
 
 The table below indicates these types:
 

test/framework/src/main/java/org/elasticsearch/test/rest/FakeRestChannel.java

@@ -24,6 +24,7 @@
 import org.elasticsearch.rest.RestStatus;
 
 import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
 
 public final class FakeRestChannel extends AbstractRestChannel {
@@ -59,4 +60,8 @@ public AtomicInteger responses() {
     public AtomicInteger errors() {
         return errors;
     }
+
+    public boolean await() throws InterruptedException {
+        return latch.await(10, TimeUnit.SECONDS);
+    }
 }

x-pack/plugin/sql/qa/debug/build.gradle

@@ -0,0 +1,14 @@
+description = 'IDE Debugging for SQL'
+
+dependencies {
+  testCompile project(path: ":client:transport")
+  testCompile project(path: ":modules:aggs-matrix-stats")
+  testCompile project(path: xpackModule('sql'))
+  testCompile project(path: xpackModule('core'))
+}
+
+forbiddenApisTest.enabled = false
+namingConventions.enabled = false
+integTest.enabled = false
+testingConventions.enabled = false
+

x-pack/plugin/sql/qa/debug/src/test/java/org/elasticsearch/xpack/sql/qa/jdbc/ClientReference.java

@@ -0,0 +1,43 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.sql.qa.jdbc;
+
+import org.elasticsearch.client.Client;
+import org.elasticsearch.common.inject.Module;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.plugins.Plugin;
+
+import java.lang.reflect.InvocationHandler;
+import java.lang.reflect.Method;
+import java.lang.reflect.Proxy;
+import java.util.Collection;
+
+import static java.util.Arrays.asList;
+
+public class ClientReference extends Plugin {
+
+    static final ClientInvocationHandler HANDLER = new ClientInvocationHandler();
+    private static final Client CLIENT =
+        (Client) Proxy.newProxyInstance(ClientReference.class.getClassLoader(), new Class[]{Client.class}, HANDLER);
+    private static final XPackLicenseState LICENSE = new XPackLicenseState(Settings.EMPTY);
+
+    @Override
+    public Collection<Module> createGuiceModules() {
+        return asList(b -> b.bind(Client.class).toInstance(CLIENT), b -> b.bind(XPackLicenseState.class).toInstance(LICENSE));
+    }
+
+    static class ClientInvocationHandler implements InvocationHandler {
+
+        volatile Client actualClient;
+
+        @Override
+        public Object invoke(Object proxy, Method method, Object[] args) throws Throwable {
+            return method.invoke(actualClient, args);
+        }
+    }
+}

x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/DebugCsvSpec.java → x-pack/plugin/sql/qa/debug/src/test/java/org/elasticsearch/xpack/sql/qa/jdbc/DebugCsvSpec.java

@@ -6,10 +6,10 @@
 package org.elasticsearch.xpack.sql.qa.jdbc;
 
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
-
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.test.junit.annotations.TestLogging;
 import org.elasticsearch.xpack.sql.qa.jdbc.CsvTestUtils.CsvTestCase;
+import org.junit.ClassRule;
 
 import java.sql.Connection;
 import java.sql.ResultSet;
@@ -20,31 +20,40 @@
 import static org.elasticsearch.xpack.sql.qa.jdbc.CsvTestUtils.executeCsvQuery;
 import static org.elasticsearch.xpack.sql.qa.jdbc.CsvTestUtils.specParser;
 
-@TestLogging("org.elasticsearch.xpack.sql:TRACE")
-public abstract class DebugCsvSpec extends SpecBaseIntegrationTestCase {
+@TestLogging(JdbcTestUtils.SQL_TRACE)
+public class DebugCsvSpec extends SpecBaseIntegrationTestCase {
+
+    @ClassRule
+    public static final EmbeddedSqlServer EMBEDDED_SERVER = new EmbeddedSqlServer();
+
     private final CsvTestCase testCase;
 
+    public DebugCsvSpec(String fileName, String groupName, String testName, Integer lineNumber, CsvTestCase testCase) {
+        super(fileName, groupName, testName, lineNumber);
+        this.testCase = testCase;
+    }
+
     @ParametersFactory(shuffle = false, argumentFormatting = SqlSpecTestCase.PARAM_FORMATTING)
     public static List<Object[]> readScriptSpec() throws Exception {
         Parser parser = specParser();
         return readScriptSpec("/debug.csv-spec", parser);
     }
 
-    public DebugCsvSpec(String fileName, String groupName, String testName, Integer lineNumber, CsvTestCase testCase) {
-        super(fileName, groupName, testName, lineNumber);
-        this.testCase = testCase;
+    @Override
+    protected final void doTest() throws Throwable {
+        try (Connection csv = csvConnection(testCase); Connection es = esJdbc()) {
+            // pass the testName as table for debugging purposes (in case the underlying reader is missing)
+            ResultSet expected = executeCsvQuery(csv, testName);
+            ResultSet elasticResults = executeJdbcQuery(es, testCase.query);
+            assertResults(expected, elasticResults);
+        }
     }
 
     @Override
-    protected void assertResults(ResultSet expected, ResultSet elastic) throws SQLException {
-        Logger log = logEsResultSet() ? logger : null;
-
-        //
-        // uncomment this to printout the result set and create new CSV tests
-        //
-        JdbcTestUtils.logResultSetMetadata(elastic, log);
-        JdbcTestUtils.logResultSetData(elastic, log);
-        //JdbcAssert.assertResultSets(expected, elastic, log);
+    public Connection esJdbc() throws SQLException {
+        // use the same random path as the rest of the tests
+        randomBoolean();
+        return EMBEDDED_SERVER.connection(connectionProperties());
     }
 
     @Override
@@ -53,12 +62,13 @@ protected boolean logEsResultSet() {
     }
 
     @Override
-    protected final void doTest() throws Throwable {
-        try (Connection csv = csvConnection(testCase); Connection es = esJdbc()) {
-            // pass the testName as table for debugging purposes (in case the underlying reader is missing)
-            ResultSet expected = executeCsvQuery(csv, testName);
-            ResultSet elasticResults = executeJdbcQuery(es, testCase.query);
-            assertResults(expected, elasticResults);
-        }
+    protected void assertResults(ResultSet expected, ResultSet elastic) throws SQLException {
+        Logger log = logEsResultSet() ? logger : null;
+
+        //
+        // uncomment this to printout the result set and create new CSV tests
+        //
+        JdbcTestUtils.logLikeCLI(elastic, log);
+        //JdbcAssert.assertResultSets(expected, elastic, log);
     }
-}
+}

x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/DebugSqlSpec.java → x-pack/plugin/sql/qa/debug/src/test/java/org/elasticsearch/xpack/sql/qa/jdbc/DebugSqlSpec.java

@@ -6,25 +6,51 @@
 package org.elasticsearch.xpack.sql.qa.jdbc;
 
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
-
+import org.apache.logging.log4j.Logger;
 import org.elasticsearch.test.junit.annotations.TestLogging;
+import org.junit.ClassRule;
 
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.SQLException;
 import java.util.List;
 
 @TestLogging(JdbcTestUtils.SQL_TRACE)
-public abstract class DebugSqlSpec extends SqlSpecTestCase {
+public class DebugSqlSpec extends SqlSpecTestCase {
+
+    @ClassRule
+    public static final EmbeddedSqlServer EMBEDDED_SERVER = new EmbeddedSqlServer();
+
+    public DebugSqlSpec(String fileName, String groupName, String testName, Integer lineNumber, String query) {
+        super(fileName, groupName, testName, lineNumber, query);
+    }
+
     @ParametersFactory(shuffle = false, argumentFormatting = PARAM_FORMATTING)
     public static List<Object[]> readScriptSpec() throws Exception {
         Parser parser = specParser();
         return readScriptSpec("/datetime.sql-spec", parser);
     }
 
-    public DebugSqlSpec(String fileName, String groupName, String testName, Integer lineNumber, String query) {
-        super(fileName, groupName, testName, lineNumber, query);
+    @Override
+    public Connection esJdbc() throws SQLException {
+        // use the same random path as the rest of the tests
+        randomBoolean();
+        return EMBEDDED_SERVER.connection(connectionProperties());
     }
 
     @Override
     protected boolean logEsResultSet() {
         return true;
     }
-}
+
+    @Override
+    protected void assertResults(ResultSet expected, ResultSet elastic) throws SQLException {
+        Logger log = logEsResultSet() ? logger : null;
+
+        //
+        // uncomment this to printout the result set and create new CSV tests
+        //
+        JdbcTestUtils.logLikeCLI(elastic, log);
+        //JdbcAssert.assertResultSets(expected, elastic, log);
+    }
+}

x-pack/plugin/sql/qa/debug/src/test/java/org/elasticsearch/xpack/sql/qa/jdbc/EmbeddedSqlServer.java

@@ -0,0 +1,105 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.sql.qa.jdbc;
+
+import org.apache.logging.log4j.LogManager;
+import org.elasticsearch.client.transport.TransportClient;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.transport.TransportAddress;
+import org.elasticsearch.search.aggregations.matrix.MatrixAggregationPlugin;
+import org.elasticsearch.transport.client.PreBuiltTransportClient;
+import org.elasticsearch.xpack.sql.plugin.SqlPlugin;
+import org.junit.rules.ExternalResource;
+
+import java.net.InetAddress;
+import java.security.AccessControlException;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.SQLException;
+import java.util.Properties;
+
+import static org.junit.Assert.assertNotNull;
+
+/**
+ * Embedded JDBC server that uses the transport client to power
+ * the jdbc endpoints in the same JVM as the tests.
+ */
+@SuppressWarnings("deprecation")
+public class EmbeddedSqlServer extends ExternalResource implements AutoCloseable {
+
+    private final Properties properties;
+    private TransportClient client;
+    private SqlHttpServer server;
+    private String jdbcUrl;
+
+    EmbeddedSqlServer() {
+        this(false);
+    }
+
+    private EmbeddedSqlServer(boolean debug) {
+        properties = new Properties();
+        if (debug) {
+            properties.setProperty("debug", "true");
+        }
+    }
+
+    @Override
+    @SuppressWarnings({"resource"})
+    protected void before() throws Throwable {
+        try {
+            Settings settings = Settings.builder()
+                .put("client.transport.ignore_cluster_name", true)
+                .build();
+            TransportAddress transportAddress = new TransportAddress(InetAddress.getLoopbackAddress(), 9300);
+            client = new PreBuiltTransportClient(settings, MatrixAggregationPlugin.class, ClientReference.class, SqlPlugin.class)
+                .addTransportAddress(transportAddress);
+
+            // update static reference
+            ClientReference.HANDLER.actualClient = client;
+        } catch (ExceptionInInitializerError e) {
+            if (e.getCause() instanceof AccessControlException) {
+                throw new RuntimeException(getClass().getSimpleName() + " is not available with the security manager", e);
+            } else {
+                throw e;
+            }
+        }
+
+        server = new SqlHttpServer(client);
+        server.start(0);
+        jdbcUrl = server.jdbcUrl();
+
+        LogManager.getLogger(EmbeddedSqlServer.class).info("Embedded SQL started at [{}]", server.url());
+    }
+
+    @Override
+    public void close() {
+        after();
+    }
+
+    @Override
+    protected void after() {
+        if (client != null) {
+            client.close();
+            client = null;
+        }
+        if (server != null) {
+            server.stop();
+            server = null;
+        }
+    }
+
+    Connection connection(Properties props) throws SQLException {
+        assertNotNull("ES JDBC Server is null - make sure ES is properly run as a @ClassRule", client);
+        Properties p = new Properties(properties);
+        p.putAll(props);
+        return DriverManager.getConnection(jdbcUrl, p);
+        //        JdbcDataSource dataSource = new JdbcDataSource();
+        //        dataSource.setProperties(properties);
+        //        dataSource.setUrl(jdbcUrl);
+        //        return dataSource.getConnection();
+    }
+}

x-pack/plugin/sql/qa/debug/src/test/java/org/elasticsearch/xpack/sql/qa/jdbc/RootHandler.java

@@ -0,0 +1,50 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.sql.qa.jdbc;
+
+import com.sun.net.httpserver.HttpExchange;
+import com.sun.net.httpserver.HttpHandler;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.common.SuppressForbidden;
+import org.elasticsearch.rest.RestStatus;
+
+import java.io.IOException;
+
+@SuppressForbidden(reason = "use http server")
+@SuppressWarnings("restriction")
+class RootHandler implements HttpHandler {
+
+    private static final Logger log = LogManager.getLogger(RootHandler.class.getName());
+
+    @Override
+    public void handle(HttpExchange http) throws IOException {
+        log.debug("Received query call...");
+
+        if ("HEAD".equals(http.getRequestMethod())) {
+            http.sendResponseHeaders(RestStatus.OK.getStatus(), 0);
+            http.close();
+            return;
+        }
+
+        fail(http, new UnsupportedOperationException("only HEAD allowed"));
+    }
+
+    private void fail(HttpExchange http, Exception ex) {
+        log.error("Caught error while transmitting response", ex);
+        try {
+            // the error conversion has failed, halt
+            if (http.getResponseHeaders().isEmpty()) {
+                http.sendResponseHeaders(RestStatus.INTERNAL_SERVER_ERROR.getStatus(), -1);
+            }
+        } catch (IOException ioEx) {
+            log.error("Caught error while trying to catch error", ex);
+        } finally {
+            http.close();
+        }
+    }
+}