OpenSearch Migration for SQL/PPL

README.md

@@ -8,12 +8,12 @@
 # Open Distro for Elasticsearch SQL
 
 
-Open Distro for Elasticsearch enables you to extract insights out of Elasticsearch using the familiar SQL query syntax. Use aggregations, group by, and where clauses to investigate your data. Read your data as JSON documents or CSV tables so you have the flexibility to use the format that works best for you.
+Open Distro for Elasticsearch enables you to extract insights out of OpenSearch using the familiar SQL query syntax. Use aggregations, group by, and where clauses to investigate your data. Read your data as JSON documents or CSV tables so you have the flexibility to use the format that works best for you.
 
 
 ## SQL Related Projects
 
-The following projects have been merged into this repository as separate folders as of July 9, 2020. Please refer to links below for details. This document will focus on the SQL plugin for Elasticsearch.
+The following projects have been merged into this repository as separate folders as of July 9, 2020. Please refer to links below for details. This document will focus on the SQL plugin for OpenSearch.
 
 * [SQL CLI](https://github.com/opendistro-for-elasticsearch/sql/tree/master/sql-cli)
 * [SQL JDBC](https://github.com/opendistro-for-elasticsearch/sql/tree/master/sql-jdbc)
@@ -33,9 +33,9 @@ Recently we have been actively improving our query engine primarily for better c
 
 ## Setup
 
-Install as plugin: build plugin from source code by following the instruction in Build section and install it to your Elasticsearch.
+Install as plugin: build plugin from source code by following the instruction in Build section and install it to your OpenSearch.
 
-After doing this, you need to restart the Elasticsearch server. Otherwise you may get errors like `Invalid index name [sql], must not start with '']; ","status":400}`.
+After doing this, you need to restart the OpenSearch server. Otherwise you may get errors like `Invalid index name [sql], must not start with '']; ","status":400}`.
 
 
 ## Build
@@ -60,7 +60,7 @@ POST https://<host>:<port>/_opendistro/_sql
 }
 ```
 
-* Explain SQL to elasticsearch query DSL
+* Explain SQL to OpenSearch query DSL
 ```
 POST _opendistro/_sql/_explain
 {
@@ -137,7 +137,7 @@ curl -XPOST https://localhost:9200/_opendistro/_sql -u admin:admin -k -d '{"quer
 
 			SELECT online FROM online GROUP BY date_range(field='insert_time','format'='yyyy-MM-dd' ,'2014-08-18','2014-08-17','now-8d','now-7d','now-6d','now')
 
-* ES Geographic
+* OpenSearch Geographic
 
 		SELECT * FROM locations WHERE GEO_BOUNDING_BOX(fieldname,100.0,1.0,101,0.0)
 
@@ -166,7 +166,7 @@ curl -XPOST https://localhost:9200/_opendistro/_sql -u admin:admin -k -d '{"quer
 *  SQL Between
 *  SQL Aliases
 *  SQL Not Null
-*  SQL(ES) Date
+*  SQL(OpenSearch) Date
 *  SQL avg()
 *  SQL count()
 *  SQL max()
@@ -190,13 +190,13 @@ Please check out JDBC driver repository for more details.
 
 ## Beyond sql features
 
-*  ES TopHits
-*  ES MISSING
-*  ES STATS
-*  ES GEO_INTERSECTS
-*  ES GEO_BOUNDING_BOX
-*  ES GEO_DISTANCE
-*  ES GEOHASH_GRID aggregation
+*  OpenSearch TopHits
+*  OpenSearch MISSING
+*  OpenSearch STATS
+*  OpenSearch GEO_INTERSECTS
+*  OpenSearch GEO_BOUNDING_BOX
+*  OpenSearch GEO_DISTANCE
+*  OpenSearch GEOHASH_GRID aggregation
 
 ## Attribution
 

build-tools/sqlplugin-coverage.gradle

@@ -13,11 +13,11 @@
  *   permissions and limitations under the License.
  */
 /**
- * ES Plugin build tools don't work with the Gradle Jacoco Plugin to report coverage out of the box.
+ * OpenSearch Plugin build tools don't work with the Gradle Jacoco Plugin to report coverage out of the box.
  * https://github.com/elastic/elasticsearch/issues/28867.
  *
- * This code sets up coverage reporting manually for ES plugin tests. This is complicated because:
- *  1. The ES integTest Task doesn't implement Gradle's JavaForkOptions so we have to manually start the jacoco agent with the test JVM
+ * This code sets up coverage reporting manually for OpenSearch plugin tests. This is complicated because:
+ *  1. The OpenSearch integTest Task doesn't implement Gradle's JavaForkOptions so we have to manually start the jacoco agent with the test JVM
  *  2. The cluster nodes are stopped using 'kill -9' which means jacoco can't dump it's execution output to a file on VM shutdown
  *  3. The Java Security Manager prevents JMX from writing execution output to the file.
  *

build.gradle

@@ -15,17 +15,23 @@
 
 buildscript {
     ext {
-        es_version = "7.10.2"
+        es_version = "7.10.3-SNAPSHOT"
     }
 
     repositories {
         mavenCentral()
-        maven { url "https://plugins.gradle.org/m2/" }
+        maven {
+            url = 's3://search-vemsarat/'
+            credentials(AwsCredentials) {
+                accessKey = System.env.AWS_ACCESS_KEY_ID ?: findProperty('aws_access_key_id')
+                secretKey = System.env.AWS_SECRET_ACCESS_KEY ?: findProperty('aws_secret_access_key')
+            }
+        }
         jcenter()
     }
 
     dependencies {
-        classpath "org.elasticsearch.gradle:build-tools:${es_version}"
+        classpath "org.opensearch.gradle:build-tools:${es_version}"
     }
 }
 
@@ -39,11 +45,11 @@ plugins {
 
 // Repository on root level is for dependencies that project code depends on. And this block must be placed after plugins{}
 repositories {
-    mavenCentral() // For Elastic Libs that you can use to get started coding until open ES libs are available
+    mavenCentral() // For Elastic Libs that you can use to get started coding until open OpenSearch libs are available
 }
 
 ext {
-    opendistroVersion = '1.13.0'
+    opendistroVersion = '1.15.0'
     isSnapshot = "true" == System.getProperty("build.snapshot", "true")
 }
 
@@ -55,6 +61,19 @@ allprojects {
     }
 }
 
+subprojects {
+    repositories {
+        mavenCentral()
+        maven {
+            url = 's3://search-vemsarat/'
+            credentials(AwsCredentials) {
+                accessKey = System.env.AWS_ACCESS_KEY_ID ?: findProperty('aws_access_key_id')
+                secretKey = System.env.AWS_SECRET_ACCESS_KEY ?: findProperty('aws_secret_access_key')
+            }
+        }
+    }
+}
+
 // TODO: fix compiler warnings
 compileJava.options.warnings = false
 compileJava {

core/src/main/java/com/amazon/opendistroforelasticsearch/sql/analysis/ExpressionAnalyzer.java

@@ -52,7 +52,6 @@
 import com.amazon.opendistroforelasticsearch.sql.expression.function.BuiltinFunctionRepository;
 import com.amazon.opendistroforelasticsearch.sql.expression.function.FunctionName;
 import com.amazon.opendistroforelasticsearch.sql.expression.window.aggregation.AggregateWindowFunction;
-import com.amazon.opendistroforelasticsearch.sql.expression.window.ranking.RankingWindowFunction;
 import com.google.common.collect.ImmutableSet;
 import java.util.ArrayList;
 import java.util.Arrays;

core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/expression/Field.java

@@ -19,7 +19,6 @@
 import com.google.common.collect.ImmutableList;
 import java.util.Collections;
 import java.util.List;
-import lombok.AllArgsConstructor;
 import lombok.EqualsAndHashCode;
 import lombok.Getter;
 import lombok.ToString;

core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/expression/Interval.java

@@ -16,7 +16,6 @@
 package com.amazon.opendistroforelasticsearch.sql.ast.expression;
 
 import com.amazon.opendistroforelasticsearch.sql.ast.AbstractNodeVisitor;
-import com.google.common.collect.ImmutableList;
 import java.util.Collections;
 import java.util.List;
 import lombok.EqualsAndHashCode;

core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/tree/Head.java

@@ -16,7 +16,6 @@
 package com.amazon.opendistroforelasticsearch.sql.ast.tree;
 
 import com.amazon.opendistroforelasticsearch.sql.ast.AbstractNodeVisitor;
-import com.amazon.opendistroforelasticsearch.sql.ast.expression.UnresolvedArgument;
 import com.google.common.collect.ImmutableList;
 import java.util.List;
 import lombok.AllArgsConstructor;

core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/tree/RelationSubquery.java

@@ -16,10 +16,8 @@
 package com.amazon.opendistroforelasticsearch.sql.ast.tree;
 
 import com.amazon.opendistroforelasticsearch.sql.ast.AbstractNodeVisitor;
-import com.amazon.opendistroforelasticsearch.sql.exception.SemanticCheckException;
 import com.google.common.collect.ImmutableList;
 import java.util.List;
-import java.util.Locale;
 import lombok.AllArgsConstructor;
 import lombok.EqualsAndHashCode;
 import lombok.RequiredArgsConstructor;

core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprBooleanValue.java

@@ -18,7 +18,6 @@
 import com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType;
 import com.amazon.opendistroforelasticsearch.sql.data.type.ExprType;
 import com.google.common.base.Objects;
-import lombok.EqualsAndHashCode;
 
 /**
  * Expression Boolean Value.

core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprLongValue.java

@@ -17,7 +17,6 @@
 
 import com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType;
 import com.amazon.opendistroforelasticsearch.sql.data.type.ExprType;
-import com.google.common.base.Objects;
 
 /**
  * Expression Long Value.

core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprValue.java

@@ -24,7 +24,6 @@
 import java.time.LocalDate;
 import java.time.LocalDateTime;
 import java.time.LocalTime;
-import java.time.ZonedDateTime;
 import java.time.temporal.TemporalAmount;
 import java.util.List;
 import java.util.Map;

core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprValueUtils.java

@@ -17,12 +17,6 @@
 
 import com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType;
 import com.amazon.opendistroforelasticsearch.sql.exception.ExpressionEvaluationException;
-import com.google.common.annotations.VisibleForTesting;
-import java.time.Instant;
-import java.time.LocalDate;
-import java.time.LocalDateTime;
-import java.time.LocalTime;
-import java.time.ZonedDateTime;
 import java.time.temporal.TemporalAmount;
 import java.util.ArrayList;
 import java.util.LinkedHashMap;

core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/aggregation/AggregatorFunction.java

@@ -15,35 +15,26 @@
 
 package com.amazon.opendistroforelasticsearch.sql.expression.aggregation;
 
-import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.ARRAY;
-import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.BOOLEAN;
 import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.DATE;
 import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.DATETIME;
 import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.DOUBLE;
 import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.FLOAT;
 import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.INTEGER;
 import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.LONG;
 import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.STRING;
-import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.STRUCT;
 import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.TIME;
 import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.TIMESTAMP;
 
 import com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType;
-import com.amazon.opendistroforelasticsearch.sql.data.type.ExprType;
 import com.amazon.opendistroforelasticsearch.sql.expression.function.BuiltinFunctionName;
 import com.amazon.opendistroforelasticsearch.sql.expression.function.BuiltinFunctionRepository;
 import com.amazon.opendistroforelasticsearch.sql.expression.function.FunctionBuilder;
 import com.amazon.opendistroforelasticsearch.sql.expression.function.FunctionName;
 import com.amazon.opendistroforelasticsearch.sql.expression.function.FunctionResolver;
 import com.amazon.opendistroforelasticsearch.sql.expression.function.FunctionSignature;
 import com.google.common.collect.ImmutableMap;
-
-import java.util.ArrayList;
 import java.util.Collections;
-import java.util.Date;
-import java.util.List;
 import java.util.stream.Collectors;
-
 import lombok.experimental.UtilityClass;
 
 /**

core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/aggregation/AvgAggregator.java

@@ -23,7 +23,6 @@
 import com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType;
 import com.amazon.opendistroforelasticsearch.sql.expression.Expression;
 import com.amazon.opendistroforelasticsearch.sql.expression.function.BuiltinFunctionName;
-import com.amazon.opendistroforelasticsearch.sql.storage.bindingtuple.BindingTuple;
 import java.util.List;
 import java.util.Locale;
 

core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/aggregation/CountAggregator.java

@@ -23,7 +23,6 @@
 import com.amazon.opendistroforelasticsearch.sql.expression.Expression;
 import com.amazon.opendistroforelasticsearch.sql.expression.aggregation.CountAggregator.CountState;
 import com.amazon.opendistroforelasticsearch.sql.expression.function.BuiltinFunctionName;
-import com.amazon.opendistroforelasticsearch.sql.storage.bindingtuple.BindingTuple;
 import java.util.List;
 import java.util.Locale;
 

core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/aggregation/MaxAggregator.java

@@ -22,7 +22,6 @@
 import com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType;
 import com.amazon.opendistroforelasticsearch.sql.expression.Expression;
 import com.amazon.opendistroforelasticsearch.sql.expression.function.BuiltinFunctionName;
-import com.amazon.opendistroforelasticsearch.sql.storage.bindingtuple.BindingTuple;
 import java.util.List;
 
 public class MaxAggregator extends Aggregator<MaxAggregator.MaxState> {

core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/aggregation/MinAggregator.java

@@ -22,7 +22,6 @@
 import com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType;
 import com.amazon.opendistroforelasticsearch.sql.expression.Expression;
 import com.amazon.opendistroforelasticsearch.sql.expression.function.BuiltinFunctionName;
-import com.amazon.opendistroforelasticsearch.sql.storage.bindingtuple.BindingTuple;
 import java.util.List;
 
 /**

core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/aggregation/NamedAggregator.java

@@ -19,11 +19,8 @@
 
 import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue;
 import com.amazon.opendistroforelasticsearch.sql.expression.ExpressionNodeVisitor;
-import com.amazon.opendistroforelasticsearch.sql.storage.bindingtuple.BindingTuple;
-import com.google.common.base.Strings;
 import lombok.EqualsAndHashCode;
 import lombok.Getter;
-import lombok.ToString;
 
 /**
  * NamedAggregator expression that represents expression with name.

core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/aggregation/SumAggregator.java

@@ -33,7 +33,6 @@
 import com.amazon.opendistroforelasticsearch.sql.expression.Expression;
 import com.amazon.opendistroforelasticsearch.sql.expression.aggregation.SumAggregator.SumState;
 import com.amazon.opendistroforelasticsearch.sql.expression.function.BuiltinFunctionName;
-import com.amazon.opendistroforelasticsearch.sql.storage.bindingtuple.BindingTuple;
 import java.util.List;
 import java.util.Locale;
 

core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/datetime/DateTimeFormatterUtil.java

@@ -3,7 +3,6 @@
 import com.amazon.opendistroforelasticsearch.sql.data.model.ExprStringValue;
 import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue;
 import com.google.common.collect.ImmutableMap;
-
 import java.time.LocalDateTime;
 import java.time.format.DateTimeFormatter;
 import java.util.Locale;

core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/operator/predicate/UnaryPredicateOperator.java

@@ -17,16 +17,12 @@
 
 import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.LITERAL_NULL;
 import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.LITERAL_TRUE;
-
 import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.BOOLEAN;
-import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.UNKNOWN;
-
 import static com.amazon.opendistroforelasticsearch.sql.expression.function.FunctionDSL.impl;
 
 import com.amazon.opendistroforelasticsearch.sql.data.model.ExprBooleanValue;
 import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue;
 import com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType;
-import com.amazon.opendistroforelasticsearch.sql.data.type.ExprType;
 import com.amazon.opendistroforelasticsearch.sql.expression.function.BuiltinFunctionName;
 import com.amazon.opendistroforelasticsearch.sql.expression.function.BuiltinFunctionRepository;
 import com.amazon.opendistroforelasticsearch.sql.expression.function.FunctionBuilder;
@@ -35,7 +31,6 @@
 import com.amazon.opendistroforelasticsearch.sql.expression.function.FunctionResolver;
 import com.amazon.opendistroforelasticsearch.sql.expression.function.FunctionSignature;
 import com.amazon.opendistroforelasticsearch.sql.expression.function.SerializableFunction;
-
 import java.util.Arrays;
 import java.util.List;
 import java.util.stream.Collectors;

core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/text/TextFunction.java

@@ -30,7 +30,6 @@
 import com.amazon.opendistroforelasticsearch.sql.expression.function.BuiltinFunctionRepository;
 import com.amazon.opendistroforelasticsearch.sql.expression.function.FunctionName;
 import com.amazon.opendistroforelasticsearch.sql.expression.function.FunctionResolver;
-
 import lombok.experimental.UtilityClass;
 
 

core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalAggregation.java

@@ -21,7 +21,6 @@
 import java.util.List;
 import lombok.EqualsAndHashCode;
 import lombok.Getter;
-import lombok.RequiredArgsConstructor;
 import lombok.ToString;
 
 /**

core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalDedupe.java

@@ -20,7 +20,6 @@
 import java.util.List;
 import lombok.EqualsAndHashCode;
 import lombok.Getter;
-import lombok.RequiredArgsConstructor;
 import lombok.ToString;
 
 /**

core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalEval.java

@@ -21,7 +21,6 @@
 import java.util.List;
 import lombok.EqualsAndHashCode;
 import lombok.Getter;
-import lombok.RequiredArgsConstructor;
 import lombok.ToString;
 import org.apache.commons.lang3.tuple.Pair;
 

core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalFilter.java

@@ -16,12 +16,9 @@
 package com.amazon.opendistroforelasticsearch.sql.planner.logical;
 
 import com.amazon.opendistroforelasticsearch.sql.expression.Expression;
-import java.util.Arrays;
 import java.util.Collections;
-import java.util.List;
 import lombok.EqualsAndHashCode;
 import lombok.Getter;
-import lombok.RequiredArgsConstructor;
 import lombok.ToString;
 
 /**