Support UDT declarations from root of schema model. (apache#7)

Related to CALCITE-5346. Previously we relied on the parser to map unrecognized datatypes to a known type using SqlAlienSystemTypeNameSpec. This worked but made it difficult to change or add new types as necessary. One would have to update at least 3 different parsers (babel, core, server) to make a change. This change allows for declaring user-defined types at the root of a schema model and allows for easy type alias mapping. These data types are shared by all schema in the model so cast and DDL expressions do not need to scope data type references to a particular sub-schema. For example: ``` inline: { version: '1.0', types: [ { name: 'BOOL', type: 'BOOLEAN' }, { name: 'BYTES', type: 'VARBINARY' }, ... ], ``` Allows for `CAST("true" as BOOL)`
olivrlee · Nov 22, 2022 · 98f9191 · 98f9191
1 parent 7a84f33
commit 98f9191
Show file tree

Hide file tree

Showing 12 changed files with 21 additions and 292 deletions.
diff --git a/babel/src/main/codegen/config.fmpp b/babel/src/main/codegen/config.fmpp
@@ -31,20 +31,11 @@ data: {
       "org.apache.calcite.sql.babel.SqlBabelCreateTable",
       "org.apache.calcite.sql.babel.TableCollectionType",
       "org.apache.calcite.sql.ddl.SqlDdlNodes",
-      # TODO(CALCITE-5346) BQ specific
-      "org.apache.calcite.sql.SqlAlienSystemTypeNameSpec",
     ]
 
     # List of new keywords. Example: "DATABASES", "TABLES". If the keyword is
     # not a reserved keyword, add it to the 'nonReservedKeywords' section.
     keywords: [
-      # TODO(CALCITE-5346) BQ specific
-      "STRING"
-      "BYTES"
-      "FLOAT64"
-      "BOOL"
-      "INT64"
-
       "IF"
       "SEMI"
       "VOLATILE"
@@ -53,13 +44,6 @@ data: {
     # List of non-reserved keywords to add;
     # items in this list become non-reserved
     nonReservedKeywordsToAdd: [
-      # TODO(CALCITE-5346) BQ specific
-      "STRING"
-      "BYTES"
-      "FLOAT64"
-      "BOOL"
-      "INT64"
-
       # not in core, added in babel
       "IF"
       "SEMI"
@@ -573,14 +557,6 @@ data: {
       "NullSafeEqual"
     ]
 
-    # List of methods for parsing custom data types.
-    # Return type of method implementation should be "SqlTypeNameSpec".
-    # Example: SqlParseTimeStampZ().
-    dataTypeParserMethods: [
-      # TODO(CALCITE-5346) BQ specific
-      "BigQuerySqlTypeNames()"
-    ]
-
     # List of files in @includes directory that have parser method
     # implementations for parsing custom SQL statements, literals or types
     # given as part of "statementParserMethods", "literalParserMethods" or

diff --git a/babel/src/main/codegen/includes/parserImpls.ftl b/babel/src/main/codegen/includes/parserImpls.ftl
@@ -208,51 +208,3 @@ void NullSafeEqual(List<Object> list, ExprContext exprContext, Span s) :
     }
     AddExpression2b(list, ExprContext.ACCEPT_SUB_QUERY)
 }
-
-/**
-* A sql type name extended basic data type, it has a counterpart basic
-* sql type name but always represents as a special alias compared with the standard name.
-* We'll want to replace this with a more generic solution at some point but this
-* satisfies our immediate need. This fix-up should not be merged upstream.
-*
-* TODO(CALCITE-5346)
-*/
-SqlTypeNameSpec BigQuerySqlTypeNames() :
-{
-    final SqlTypeName typeName;
-    final String typeAlias;
-    int precision = -1;
-}
-{
-    (
-        <BOOL> {
-            typeName = SqlTypeName.BOOLEAN;
-            typeAlias = token.image;
-        }
-    |
-        <BYTES> {
-            typeName = SqlTypeName.VARBINARY;
-            typeAlias = token.image;
-            precision = Integer.MAX_VALUE;
-        }
-    |
-        <FLOAT64> {
-            typeName = SqlTypeName.DOUBLE;
-            typeAlias = token.image;
-        }
-    |
-        <INT64> {
-            typeName = SqlTypeName.BIGINT;
-            typeAlias = token.image;
-        }
-    |
-        <STRING> {
-            typeName = SqlTypeName.VARCHAR;
-            typeAlias = token.image;
-            precision = Integer.MAX_VALUE;
-        }
-    )
-    {
-        return new SqlAlienSystemTypeNameSpec(typeAlias, typeName, precision, getPos());
-    }
-}
diff --git a/core/src/main/codegen/default_config.fmpp b/core/src/main/codegen/default_config.fmpp
@@ -26,23 +26,10 @@ parser: {
   # List of new keywords. Example: "DATABASES", "TABLES". If the keyword is
   # not a reserved keyword, add it to the 'nonReservedKeywords' section.
   keywords: [
-    # TODO(CALCITE-5346) BQ specific
-    "STRING"
-    "BYTES"
-    "FLOAT64"
-    "BOOL"
-    "INT64"
   ]
 
   # List of keywords from "keywords" section that are not reserved.
   nonReservedKeywords: [
-    # TODO(CALCITE-5346) BQ specific
-    "STRING"
-    "BYTES"
-    "FLOAT64"
-    "BOOL"
-    "INT64"
-
     "A"
     "ABSENT"
     "ABSOLUTE"
@@ -400,8 +387,6 @@ parser: {
   # Return type of method implementation should be "SqlTypeNameSpec".
   # Example: SqlParseTimeStampZ().
   dataTypeParserMethods: [
-     # TODO(CALCITE-5346) BQ specific
-    "BigQuerySqlTypeNames()"
   ]
 
   # List of methods for parsing builtin function calls.
@@ -444,7 +429,6 @@ parser: {
   # "dataTypeParserMethods".
   # Example: "parserImpls.ftl".
   implementationFiles: [
-    "parserImpls.ftl"
   ]
 
   includePosixOperators: false

diff --git a/core/src/main/codegen/includes/parserImpls.ftl b/core/src/main/codegen/includes/parserImpls.ftl
@@ -32,51 +32,3 @@
     }
   }
 -->
-
-/**
-* A sql type name extended basic data type, it has a counterpart basic
-* sql type name but always represents as a special alias compared with the standard name.
-* We'll want to replace this with a more generic solution at some point but this
-* satisfies our immediate need. This fix-up should not be merged upstream.
-*
-* TODO(CALCITE-5346)
-*/
-SqlTypeNameSpec BigQuerySqlTypeNames() :
-{
-    final SqlTypeName typeName;
-    final String typeAlias;
-    int precision = -1;
-}
-{
-    (
-        <BOOL> {
-            typeName = SqlTypeName.BOOLEAN;
-            typeAlias = token.image;
-        }
-    |
-        <BYTES> {
-            typeName = SqlTypeName.VARBINARY;
-            typeAlias = token.image;
-            precision = Integer.MAX_VALUE;
-        }
-    |
-        <FLOAT64> {
-            typeName = SqlTypeName.DOUBLE;
-            typeAlias = token.image;
-        }
-    |
-        <INT64> {
-            typeName = SqlTypeName.BIGINT;
-            typeAlias = token.image;
-        }
-    |
-        <STRING> {
-            typeName = SqlTypeName.VARCHAR;
-            typeAlias = token.image;
-            precision = Integer.MAX_VALUE;
-        }
-    )
-    {
-        return new SqlAlienSystemTypeNameSpec(typeAlias, typeName, precision, getPos());
-    }
-}
diff --git a/core/src/main/java/org/apache/calcite/model/JsonRoot.java b/core/src/main/java/org/apache/calcite/model/JsonRoot.java
@@ -73,6 +73,12 @@ public class JsonRoot {
    */
   public final List<JsonSchema> schemas = new ArrayList<>();
 
+  /** Types in the root schema. Shared by all schemas in the model.
+   *
+   * <p>The list may be empty.
+   */
+  public final List<JsonType> types = new ArrayList<>();
+
   @JsonCreator
   public JsonRoot(
       @JsonProperty(value = "version", required = true) String version,

diff --git a/core/src/main/java/org/apache/calcite/model/ModelHandler.java b/core/src/main/java/org/apache/calcite/model/ModelHandler.java
@@ -196,6 +196,9 @@ public void visit(JsonRoot jsonRoot) {
     final Pair<@Nullable String, SchemaPlus> pair =
         Pair.of(null, connection.getRootSchema());
     schemaStack.push(pair);
+    for (JsonType rootType : jsonRoot.types) {
+      rootType.accept(this);
+    }
     for (JsonSchema schema : jsonRoot.schemas) {
       schema.accept(this);
     }

diff --git a/core/src/test/codegen/config.fmpp b/core/src/test/codegen/config.fmpp
@@ -39,11 +39,6 @@ data: {
     keywords: [
       "UPLOAD"
       "JAR"
-      "STRING"
-      "INT64"
-      "FLOAT64"
-      "BOOL"
-      "BYTES"
     ]
 
     # List of methods for parsing custom SQL statements.
@@ -68,10 +63,6 @@ data: {
       "SqlCreateView"
     ]
 
-    dataTypeParserMethods: [
-      "BigQuerySqlTypeNames()"
-    ]
-
     # List of files in @includes directory that have parser method
     # implementations for parsing custom SQL statements, literals or types
     # given as part of "statementParserMethods", "literalParserMethods" or

diff --git a/core/src/test/codegen/includes/parserImpls.ftl b/core/src/test/codegen/includes/parserImpls.ftl
@@ -80,51 +80,3 @@ SqlNode SqlDescribeSpacePower() :
         return null;
     }
 }
-
-/**
-* A sql type name extended basic data type, it has a counterpart basic
-* sql type name but always represents as a special alias compared with the standard name.
-* We'll want to replace this with a more generic solution at some point but this
-* satisfies our immediate need. This fix-up should not be merged upstream.
-*
-* For tracking https://issues.apache.org/jira/browse/CALCITE-5346
-*/
-SqlTypeNameSpec BigQuerySqlTypeNames() :
-{
-    final SqlTypeName typeName;
-    final String typeAlias;
-    int precision = -1;
-}
-{
-    (
-        <BOOL> {
-            typeName = SqlTypeName.BOOLEAN;
-            typeAlias = token.image;
-        }
-    |
-        <BYTES> {
-            typeName = SqlTypeName.VARBINARY;
-            typeAlias = token.image;
-            precision = Integer.MAX_VALUE;
-        }
-    |
-        <FLOAT64> {
-            typeName = SqlTypeName.DOUBLE;
-            typeAlias = token.image;
-        }
-    |
-        <INT64> {
-            typeName = SqlTypeName.BIGINT;
-            typeAlias = token.image;
-        }
-    |
-        <STRING> {
-            typeName = SqlTypeName.VARCHAR;
-            typeAlias = token.image;
-            precision = Integer.MAX_VALUE;
-        }
-    )
-    {
-        return new SqlAlienSystemTypeNameSpec(typeAlias, typeName, precision, getPos());
-    }
-}
diff --git a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java
@@ -1776,40 +1776,6 @@ private SqlDialect nonOrdinalDialect() {
     sql(timeTrunc).withLibrary(SqlLibrary.BIG_QUERY).ok(expectedTimeTrunc);
   }
 
-  /**
-   * Test casts to BQ specific datatypes. To make this work we added an extension
-   * to the base parser in Parser.jj. We'll want to replace this with a more
-   * generic solution at some point but this satisfies our immediate need.
-   *
-   * For tracking https://issues.apache.org/jira/browse/CALCITE-5346
-   */
-  @Test void testBigQueryDataTypes() {
-    final String int64 = "select cast(34.55 as INT64)\n"
-        + "from \"foodmart\".\"product\"\n";
-    final String expectedInt64 = "SELECT 34\n"
-        + "FROM \"foodmart\".\"product\"";
-
-    final String float64 = "select cast(34.1 as float64)\n"
-        + "from \"foodmart\".\"product\"\n";
-    final String expectedFloat64 = "SELECT 34.1\n"
-        + "FROM \"foodmart\".\"product\"";
-
-    final String bool = "select cast(0 as bool)\n"
-        + "from \"foodmart\".\"product\"\n";
-    final String expectedBool = "SELECT FALSE\n"
-        + "FROM \"foodmart\".\"product\"";
-
-    final String s = "select cast(4 as string)\n"
-        + "from \"foodmart\".\"product\"\n";
-    final String expectedString = "SELECT '4'\n"
-        + "FROM \"foodmart\".\"product\"";
-
-    sql(int64).withLibrary(SqlLibrary.BIG_QUERY).ok(expectedInt64);
-    sql(float64).withLibrary(SqlLibrary.BIG_QUERY).ok(expectedFloat64);
-    sql(bool).withLibrary(SqlLibrary.BIG_QUERY).ok(expectedBool);
-    sql(s).withLibrary(SqlLibrary.BIG_QUERY).ok(expectedString);
-  }
-
   /** Test case for
    * <a href="https://issues.apache.org/jira/browse/CALCITE-3220">[CALCITE-3220]
    * HiveSqlDialect should transform the SQL-standard TRIM function to TRIM,

diff --git a/core/src/test/java/org/apache/calcite/test/UdtTest.java b/core/src/test/java/org/apache/calcite/test/UdtTest.java
@@ -25,6 +25,12 @@ class UdtTest {
   private CalciteAssert.AssertThat withUdt() {
     final String model = "{\n"
         + "  version: '1.0',\n"
+        + "  types: [\n"
+        + "     {\n"
+        + "       name: 'foo',\n"
+        + "       type: 'BIGINT'\n"
+        + "     }"
+        + "   ],\n"
         + "   schemas: [\n"
         + "     {\n"
         + "       name: 'adhoc',\n"
@@ -59,6 +65,12 @@ private CalciteAssert.AssertThat withUdt() {
     withUdt().query(sql).returns("LD=1\n");
   }
 
+  @Test void testRootUdt() {
+    final String sql = "select CAST(\"id\" AS foo) as ld "
+        + "from (VALUES ROW(1, 'SameName')) AS \"t\" (\"id\", \"desc\")";
+    withUdt().query(sql).returns("LD=1\n");
+  }
+
   /** Test case for
    * <a href="https://issues.apache.org/jira/browse/CALCITE-3045">[CALCITE-3045]
    * NullPointerException when casting null literal to composite user defined type</a>. */

diff --git a/server/src/main/codegen/config.fmpp b/server/src/main/codegen/config.fmpp
@@ -30,20 +30,11 @@ data: {
       "org.apache.calcite.sql.SqlCreate"
       "org.apache.calcite.sql.SqlDrop"
       "org.apache.calcite.sql.ddl.SqlDdlNodes"
-      # TODO(CALCITE-5346) BQ specific
-      "org.apache.calcite.sql.SqlAlienSystemTypeNameSpec"
     ]
 
     # List of new keywords. Example: "DATABASES", "TABLES". If the keyword is
     # not a reserved keyword, add it to the 'nonReservedKeywords' section.
     keywords: [
-      # TODO(CALCITE-5346) BQ specific
-      "STRING"
-      "BYTES"
-      "FLOAT64"
-      "BOOL"
-      "INT64"
-
       "IF"
       "MATERIALIZED"
       "STORED"
@@ -91,14 +82,6 @@ data: {
       "SqlDropFunction"
     ]
 
-    # List of methods for parsing custom data types.
-    # Return type of method implementation should be "SqlTypeNameSpec".
-    # Example: SqlParseTimeStampZ().
-    dataTypeParserMethods: [
-      # TODO(CALCITE-5346) BQ specific
-      "BigQuerySqlTypeNames()"
-    ]
-
     # List of files in @includes directory that have parser method
     # implementations for parsing custom SQL statements, literals or types
     # given as part of "statementParserMethods", "literalParserMethods" or