Upgrade Calcite to 1.30 and adapt it for Kylin

For SDK dialect conversion

Calcite 1.30 changed Rexcall operator from SqlCaseOperator to SqlPostfixOperator in RelOptUtil#isDistinctFromInternal

Calcite 1.30 remove EquiJoinInfo and NonEquiJoinInfo, but we need rexbuilder

Calcite 1.30 implement CHAR function

Fallback the logic of the simplification condition to version 1.16, otherwise it may lead to a failure to prune partitions

Revert "Fallback the logic of the simplification condition to version 1.16, otherwise it may lead to a failure to prune partitions"

This reverts commit 27f1a28.

[Follow up] Fallback the logic of the simplification condition to version 1.16, otherwise it may lead to a failure to prune partitions

Refer to KE-36291 for an adapted method to simplify the filter condition

Refer to AL-5295 fix the ut about CharNColumnTest#testCharNColumn

Calcite 1.30 fix about SumCaseWhenFunctionRule and CountDistinctCaseWhenFunctionRule

[Follow up] Fix ProjectRel replacement and old agg replacement caused by RelBuilder#aggregate method

Roll back Calcite's computeDigest and simplifyCase logic from previous versions to ensure correct matching of Kylin models

[Follow up] Fix the way digest is calculated, delete the previous logic

Fix exception in Calcite 1.30 where data type conversion affected logical plan changes

Fix with Sort optimize and Trim function

Revert "Calcite 1.30 implement CHAR function"

This reverts commit 0e1f6b0.

[CALCITE-5241] Implement CHAR function for MySQL and Spark, also JDBC '{fn CHAR(n)}'

Close apache#2878

Debug for Calcite deploy

Calcite 1.30 don't keep the precision of BigDecimal, add scale to fix exception

Fix SqlBasicCall's Deep Copy Logic Raises Rule Optimization Exception and TDVT test

Change kap-external-guava20 to kylin-external-guava30 and package name

Remove the default Unicode operation on the quoteStringLiteral function in Calcite 1.30

.gitignore

@@ -56,3 +56,27 @@ nb-configuration.xml
 
 # Local configuration file (sdk path, etc)
 local.properties
+
+# Generate files
+babel/bin/
+cassandra/bin/
+core/bin/
+druid/bin/
+elasticsearch/bin/
+example/csv/bin/
+example/function/bin/
+file/bin/
+geode/bin/
+innodb/bin/
+kafka/bin/
+linq4j/bin/
+mongodb/bin/
+pig/bin/
+piglet/bin/
+plus/bin/
+redis/bin/
+server/bin/
+spark/bin/
+splunk/bin/
+testkit/bin/
+ubenchmark/bin/

babel/build.gradle.kts

@@ -26,7 +26,7 @@ dependencies {
     api(project(":core"))
     api("org.apache.calcite.avatica:avatica-core")
 
-    implementation("com.google.guava:guava")
+    implementation("org.apache.kylin:kylin-external-guava30")
     implementation("org.slf4j:slf4j-api")
 
     testImplementation("net.hydromatic:quidem")

babel/src/test/java/org/apache/calcite/test/BabelParserTest.java

@@ -26,7 +26,7 @@
 import org.apache.calcite.sql.parser.babel.SqlBabelParserImpl;
 import org.apache.calcite.tools.Hoist;
 
-import com.google.common.base.Throwables;
+import org.apache.kylin.guava30.shaded.common.base.Throwables;
 
 import org.checkerframework.checker.nullness.qual.Nullable;
 import org.junit.jupiter.api.Disabled;

babel/src/test/java/org/apache/calcite/test/BabelQuidemTest.java

@@ -30,8 +30,8 @@
 import org.apache.calcite.tools.Frameworks;
 import org.apache.calcite.tools.Planner;
 
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableSet;
+import org.apache.kylin.guava30.shaded.common.collect.ImmutableList;
+import org.apache.kylin.guava30.shaded.common.collect.ImmutableSet;
 
 import net.hydromatic.quidem.AbstractCommand;
 import net.hydromatic.quidem.Command;

bom/build.gradle.kts

@@ -61,7 +61,7 @@ dependencies {
         apiv("com.github.stephenc.jcip:jcip-annotations")
         apiv("com.google.errorprone:error_prone_annotations", "errorprone")
         apiv("com.google.errorprone:error_prone_type_annotations", "errorprone")
-        apiv("com.google.guava:guava")
+        apiv("org.apache.kylin:kylin-external-guava30")
         apiv("com.google.protobuf:protobuf-java", "protobuf")
         apiv("com.google.uzaygezen:uzaygezen-core", "uzaygezen")
         apiv("com.h2database:h2")

build.gradle.kts

@@ -57,9 +57,25 @@ plugins {
     id("com.autonomousapps.dependency-analysis") apply false
 }
 
-repositories {
-    // At least for RAT
-    mavenCentral()
+// define repo url
+val snapshotsRepoUrl = uri("https://repo-ofs.kyligence.com/repository/maven-snapshots/")
+val releasesRepoUrl = uri("https://repo-ofs.kyligence.com/repository/maven-releases/")
+
+allprojects {
+    repositories {
+        // At least for RAT
+        mavenLocal()
+        // achieve dependencies
+        maven {
+            name = "snapshots"
+            url = snapshotsRepoUrl
+        }
+        maven {
+            name = "releases"
+            url = releasesRepoUrl
+        }
+        mavenCentral()
+    }
 }
 
 fun reportsForHumans() = !(System.getenv()["CI"]?.toBoolean() ?: false)
@@ -887,6 +903,16 @@ allprojects {
                             url.set("https://github.com/apache/calcite")
                             tag.set("HEAD")
                         }
+                        repositories {
+                            val finalUrl = if (version.toString().endsWith("SNAPSHOT")) snapshotsRepoUrl else releasesRepoUrl
+                            maven {
+                                url = finalUrl
+                                credentials {
+                                    username = "yaguang.jia"
+                                    password = "Qwer1234,"
+                                }
+                            }
+                        }
                     }
                 }
             }

cassandra/build.gradle.kts

@@ -26,7 +26,7 @@ dependencies {
     api(project(":linq4j"))
 
     api("com.datastax.oss:java-driver-core")
-    api("com.google.guava:guava")
+    api("org.apache.kylin:kylin-external-guava30")
     api("org.slf4j:slf4j-api")
 
     implementation("org.apache.calcite.avatica:avatica-core")

cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CassandraMethod.java

@@ -18,7 +18,7 @@
 
 import org.apache.calcite.linq4j.tree.Types;
 
-import com.google.common.collect.ImmutableMap;
+import org.apache.kylin.guava30.shaded.common.collect.ImmutableMap;
 
 import java.lang.reflect.Method;
 import java.util.List;

cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CassandraProject.java

@@ -27,7 +27,7 @@
 import org.apache.calcite.rex.RexNode;
 import org.apache.calcite.util.Pair;
 
-import com.google.common.collect.ImmutableList;
+import org.apache.kylin.guava30.shaded.common.collect.ImmutableList;
 
 import org.checkerframework.checker.nullness.qual.Nullable;
 

cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CassandraSchema.java

@@ -40,6 +40,8 @@
 import org.apache.calcite.util.Pair;
 import org.apache.calcite.util.trace.CalciteTrace;
 
+import org.apache.kylin.guava30.shaded.common.collect.ImmutableMap;
+
 import com.datastax.oss.driver.api.core.CqlIdentifier;
 import com.datastax.oss.driver.api.core.CqlSession;
 import com.datastax.oss.driver.api.core.cql.Row;
@@ -54,7 +56,6 @@
 import com.datastax.oss.driver.api.core.type.MapType;
 import com.datastax.oss.driver.api.core.type.SetType;
 import com.datastax.oss.driver.api.core.type.TupleType;
-import com.google.common.collect.ImmutableMap;
 
 import org.slf4j.Logger;
 

cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CassandraSchemaFactory.java

@@ -21,8 +21,9 @@
 import org.apache.calcite.schema.SchemaPlus;
 import org.apache.calcite.util.trace.CalciteTrace;
 
+import org.apache.kylin.guava30.shaded.common.collect.ImmutableSet;
+
 import com.datastax.oss.driver.api.core.CqlSession;
-import com.google.common.collect.ImmutableSet;
 
 import org.slf4j.Logger;
 

cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CassandraTable.java

@@ -38,9 +38,10 @@
 import org.apache.calcite.sql.type.SqlTypeFactoryImpl;
 import org.apache.calcite.util.Util;
 
+import org.apache.kylin.guava30.shaded.common.collect.ImmutableList;
+
 import com.datastax.oss.driver.api.core.CqlSession;
 import com.datastax.oss.driver.api.core.cql.ResultSet;
-import com.google.common.collect.ImmutableList;
 
 import org.checkerframework.checker.nullness.qual.Nullable;
 

cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CassandraTableScan.java

@@ -25,7 +25,7 @@
 import org.apache.calcite.rel.core.TableScan;
 import org.apache.calcite.rel.type.RelDataType;
 
-import com.google.common.collect.ImmutableList;
+import org.apache.kylin.guava30.shaded.common.collect.ImmutableList;
 
 import org.checkerframework.checker.nullness.qual.Nullable;
 

cassandra/src/main/java/org/apache/calcite/adapter/cassandra/CqlToSqlTypeConversionRules.java

@@ -18,9 +18,10 @@
 
 import org.apache.calcite.sql.type.SqlTypeName;
 
+import org.apache.kylin.guava30.shaded.common.collect.ImmutableMap;
+
 import com.datastax.oss.driver.api.core.type.DataType;
 import com.datastax.oss.driver.api.core.type.DataTypes;
-import com.google.common.collect.ImmutableMap;
 
 import java.util.Map;
 

cassandra/src/test/java/org/apache/calcite/test/CassandraAdapterDataTypesTest.java

@@ -18,9 +18,10 @@
 
 import org.apache.calcite.avatica.util.DateTimeUtils;
 
+import org.apache.kylin.guava30.shaded.common.collect.ImmutableMap;
+
 import com.datastax.oss.driver.api.core.CqlSession;
 import com.datastax.oss.driver.api.core.type.codec.TypeCodecs;
-import com.google.common.collect.ImmutableMap;
 
 import org.cassandraunit.CQLDataLoader;
 import org.cassandraunit.dataset.cql.ClassPathCQLDataSet;

cassandra/src/test/java/org/apache/calcite/test/CassandraAdapterTest.java

@@ -16,8 +16,9 @@
  */
 package org.apache.calcite.test;
 
+import org.apache.kylin.guava30.shaded.common.collect.ImmutableMap;
+
 import com.datastax.oss.driver.api.core.CqlSession;
-import com.google.common.collect.ImmutableMap;
 
 import org.cassandraunit.CQLDataLoader;
 import org.cassandraunit.dataset.cql.ClassPathCQLDataSet;

cassandra/src/test/java/org/apache/calcite/test/CassandraExtension.java

@@ -27,9 +27,9 @@
 import org.apache.cassandra.service.CassandraDaemon;
 import org.apache.cassandra.service.StorageService;
 import org.apache.cassandra.utils.FBUtilities;
+import org.apache.kylin.guava30.shaded.common.collect.ImmutableMap;
 
 import com.datastax.oss.driver.api.core.CqlSession;
-import com.google.common.collect.ImmutableMap;
 
 import org.cassandraunit.utils.EmbeddedCassandraServerHelper;
 import org.junit.jupiter.api.extension.ConditionEvaluationResult;

core/build.gradle.kts

@@ -49,7 +49,7 @@ dependencies {
     api("com.esri.geometry:esri-geometry-api")
     api("com.fasterxml.jackson.core:jackson-annotations")
     api("com.google.errorprone:error_prone_annotations")
-    api("com.google.guava:guava")
+    api("org.apache.kylin:kylin-external-guava30")
     api("org.apache.calcite.avatica:avatica-core")
     api("org.apiguardian:apiguardian-api")
     api("org.checkerframework:checker-qual")

core/src/main/codegen/templates/Parser.jj

@@ -6919,6 +6919,7 @@ SqlIdentifier ReservedFunctionName() :
     |   <AVG>
     |   <CARDINALITY>
     |   <CEILING>
+    |   <CHAR>
     |   <CHAR_LENGTH>
     |   <CHARACTER_LENGTH>
     |   <COALESCE>

core/src/main/java/org/apache/calcite/DataContext.java

@@ -23,7 +23,7 @@
 import org.apache.calcite.schema.SchemaPlus;
 import org.apache.calcite.sql.advise.SqlAdvisor;
 
-import com.google.common.base.CaseFormat;
+import org.apache.kylin.guava30.shaded.common.base.CaseFormat;
 
 import org.checkerframework.checker.nullness.qual.Nullable;
 

core/src/main/java/org/apache/calcite/DataContexts.java

@@ -21,7 +21,7 @@
 import org.apache.calcite.linq4j.QueryProvider;
 import org.apache.calcite.schema.SchemaPlus;
 
-import com.google.common.collect.ImmutableMap;
+import org.apache.kylin.guava30.shaded.common.collect.ImmutableMap;
 
 import org.checkerframework.checker.nullness.qual.Nullable;
 

core/src/main/java/org/apache/calcite/adapter/clone/ArrayTable.java

@@ -38,8 +38,8 @@
 import org.apache.calcite.util.ImmutableBitSet;
 import org.apache.calcite.util.Pair;
 
-import com.google.common.base.Supplier;
-import com.google.common.collect.ImmutableList;
+import org.apache.kylin.guava30.shaded.common.base.Supplier;
+import org.apache.kylin.guava30.shaded.common.collect.ImmutableList;
 
 import org.checkerframework.checker.nullness.qual.Nullable;
 

core/src/main/java/org/apache/calcite/adapter/clone/CloneSchema.java

@@ -34,8 +34,8 @@
 import org.apache.calcite.schema.Table;
 import org.apache.calcite.schema.impl.AbstractSchema;
 
-import com.google.common.base.Suppliers;
-import com.google.common.collect.ImmutableList;
+import org.apache.kylin.guava30.shaded.common.base.Suppliers;
+import org.apache.kylin.guava30.shaded.common.collect.ImmutableList;
 
 import org.checkerframework.checker.nullness.qual.Nullable;
 

core/src/main/java/org/apache/calcite/adapter/clone/ListTable.java

@@ -30,7 +30,7 @@
 import org.apache.calcite.schema.Statistic;
 import org.apache.calcite.schema.Statistics;
 
-import com.google.common.collect.ImmutableList;
+import org.apache.kylin.guava30.shaded.common.collect.ImmutableList;
 
 import java.lang.reflect.Type;
 import java.util.Iterator;

core/src/main/java/org/apache/calcite/adapter/enumerable/EnumUtils.java

@@ -55,8 +55,8 @@
 import org.apache.calcite.util.Pair;
 import org.apache.calcite.util.Util;
 
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableMap;
+import org.apache.kylin.guava30.shaded.common.collect.ImmutableList;
+import org.apache.kylin.guava30.shaded.common.collect.ImmutableMap;
 
 import org.checkerframework.checker.nullness.qual.Nullable;
 
@@ -335,8 +335,13 @@ static Type fromInternal(Type type) {
    * is no need to convert
    */
   public static Expression convert(Expression operand, Type toType) {
+    return convert(operand, toType, -1);
+  }
+
+  // Calcite 1.30 don't keep the precision of BigDecimal, This will cause calculate error
+  public static Expression convert(Expression operand, Type toType, int scale) {
     final Type fromType = operand.getType();
-    return convert(operand, fromType, toType);
+    return convert(operand, fromType, toType, scale);
   }
 
   /**
@@ -350,6 +355,12 @@ public static Expression convert(Expression operand, Type toType) {
    */
   public static Expression convert(Expression operand, Type fromType,
       Type toType) {
+    return convert(operand, fromType, toType, -1);
+  }
+
+  // Calcite 1.30 don't keep the precision of BigDecimal, This will cause calculate error
+  public static Expression convert(Expression operand, Type fromType,
+      Type toType, int scale) {
     if (!Types.needTypeCast(fromType, toType)) {
       return operand;
     }
@@ -487,7 +498,8 @@ public static Expression convert(Expression operand, Type fromType,
       if (fromPrimitive != null) {
         // E.g. from "int" to "BigDecimal".
         // Generate "new BigDecimal(x)"
-        return Expressions.new_(BigDecimal.class, operand);
+        // // Calcite 1.30 don't keep the precision of BigDecimal, This will cause calculate error
+        return Expressions.new_(BigDecimal.class, scale, operand);
       }
       // E.g. from "Object" to "BigDecimal".
       // Generate "x == null ? null : SqlFunctions.toBigDecimal(x)"

core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableAggregate.java

@@ -34,7 +34,7 @@
 import org.apache.calcite.util.BuiltInMethod;
 import org.apache.calcite.util.ImmutableBitSet;
 
-import com.google.common.collect.ImmutableList;
+import org.apache.kylin.guava30.shaded.common.collect.ImmutableList;
 
 import org.checkerframework.checker.nullness.qual.Nullable;
 

core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableAggregateBase.java

@@ -44,7 +44,7 @@
 import org.apache.calcite.util.Pair;
 import org.apache.calcite.util.Util;
 
-import com.google.common.collect.ImmutableList;
+import org.apache.kylin.guava30.shaded.common.collect.ImmutableList;
 
 import org.checkerframework.checker.nullness.qual.Nullable;
 

core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableBatchNestedLoopJoin.java

@@ -40,7 +40,7 @@
 import org.apache.calcite.util.ImmutableBitSet;
 import org.apache.calcite.util.Pair;
 
-import com.google.common.collect.ImmutableList;
+import org.apache.kylin.guava30.shaded.common.collect.ImmutableList;
 
 import org.checkerframework.checker.nullness.qual.Nullable;
 

core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableBindable.java

@@ -33,7 +33,7 @@
 import org.apache.calcite.runtime.ArrayBindable;
 import org.apache.calcite.runtime.Bindable;
 
-import com.google.common.collect.ImmutableMap;
+import org.apache.kylin.guava30.shaded.common.collect.ImmutableMap;
 
 import org.checkerframework.checker.nullness.qual.Nullable;
 

core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableCalc.java

@@ -49,7 +49,7 @@
 import org.apache.calcite.util.Pair;
 import org.apache.calcite.util.Util;
 
-import com.google.common.collect.ImmutableList;
+import org.apache.kylin.guava30.shaded.common.collect.ImmutableList;
 
 import org.checkerframework.checker.nullness.qual.Nullable;