draft commit v9: msq quidem tests

extensions-core/multi-stage-query/pom.xml

@@ -213,6 +213,32 @@
             <artifactId>junit</artifactId>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>net.hydromatic</groupId>
+            <artifactId>quidem</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.calcite</groupId>
+            <artifactId>calcite-testkit</artifactId>
+            <version>${calcite.version}</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>org.junit.jupiter</groupId>
+                    <artifactId>*</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.hamcrest</groupId>
+                    <artifactId>*</artifactId>
+                </exclusion>
+            </exclusions>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.reflections</groupId>
+            <artifactId>reflections</artifactId>
+            <scope>test</scope>
+        </dependency>
         <dependency>
             <groupId>org.junit.jupiter</groupId>
             <artifactId>junit-jupiter-api</artifactId>

extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQDrillWindowQueryTest.java

@@ -23,15 +23,18 @@
 import com.google.common.collect.ImmutableList;
 import com.google.inject.Injector;
 import com.google.inject.Module;
+import org.apache.druid.client.indexing.NoopOverlordClient;
 import org.apache.druid.guice.DruidInjectorBuilder;
 import org.apache.druid.msq.exec.MSQDrillWindowQueryTest.DrillWindowQueryMSQComponentSupplier;
+import org.apache.druid.msq.guice.MSQSqlModule;
 import org.apache.druid.msq.sql.MSQTaskSqlEngine;
 import org.apache.druid.msq.test.CalciteMSQTestsHelper;
 import org.apache.druid.msq.test.ExtractResultsFactory;
 import org.apache.druid.msq.test.MSQTestOverlordServiceClient;
 import org.apache.druid.msq.test.MSQTestTaskActionClient;
 import org.apache.druid.msq.test.VerifyMSQSupportedNativeQueriesPredicate;
 import org.apache.druid.query.groupby.TestGroupByBuffers;
+import org.apache.druid.rpc.indexing.OverlordClient;
 import org.apache.druid.server.QueryLifecycleFactory;
 import org.apache.druid.sql.calcite.DrillWindowQueryTest;
 import org.apache.druid.sql.calcite.QueryTestBuilder;
@@ -55,6 +58,8 @@ public void configureGuice(DruidInjectorBuilder builder)
     {
       super.configureGuice(builder);
       builder.addModules(CalciteMSQTestsHelper.fetchModules(tempDirProducer::newTempFolder, TestGroupByBuffers.createDefault()).toArray(new Module[0]));
+      builder.addModule(new MSQSqlModule());
+      builder.addModule(binder -> binder.bind(OverlordClient.class).to(NoopOverlordClient.class));
     }
 
     @Override
@@ -93,10 +98,10 @@ protected QueryTestBuilder testBuilder()
         .verifyNativeQueries(new VerifyMSQSupportedNativeQueriesPredicate());
   }
 
-//  @DrillTest("frameclause/subQueries/frmInSubQry_53")
-//  @Test
-//  public void test_wikipedia_1()
-//  {
-//    windowQueryTest();
-//  }
+  @DrillTest("frameclause/subQueries/frmInSubQry_53")
+  @Test
+  public void test_wikipedia_1()
+  {
+    windowQueryTest();
+  }
 }

extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQQuidemTest.java

@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.msq.exec;
+
+import org.apache.druid.common.config.NullHandling;
+import org.apache.druid.quidem.DruidQuidemTestBase;
+import org.apache.druid.quidem.ProjectPathUtils;
+import org.junit.jupiter.api.condition.EnabledIf;
+
+import java.io.File;
+
+@EnabledIf(value = "enabled", disabledReason = "These tests are only run in SqlCompatible mode!")
+public class MSQQuidemTest extends DruidQuidemTestBase
+{
+  public static boolean enabled()
+  {
+    NullHandling.initializeForTests();
+    return NullHandling.sqlCompatible();
+  }
+
+  public MSQQuidemTest()
+  {
+    super();
+  }
+
+  @Override
+  protected File getTestRoot()
+  {
+    return ProjectPathUtils.getPathFromProjectRoot("extensions-core/multi-stage-query/src/test/quidem/" + getClass().getName());
+  }
+}

extensions-core/multi-stage-query/src/test/quidem/org.apache.druid.msq.exec.MSQQuidemTest/decoupled.iq

@@ -0,0 +1,96 @@
+!set plannerStrategy DECOUPLED
+!use druidtest://?componentSupplier=DrillWindowQueryMSQComponentSupplier
+!set outputformat mysql
+
+select cityName, count(case when delta > 0 then channel end) as cnt, count(1) as aall
+from wikipedia
+where cityName in ('New York', 'Aarhus')
+group by 1
+order by 1;
++----------+-----+------+
+| cityName | cnt | aall |
++----------+-----+------+
+| Aarhus   |   0 |    1 |
+| New York |   7 |   13 |
++----------+-----+------+
+(2 rows)
+
+!ok
+LogicalSort(sort0=[$0], dir0=[ASC])
+  LogicalAggregate(group=[{0}], cnt=[COUNT($1)], aall=[COUNT()])
+    LogicalProject(cityName=[$2], $f1=[CASE(>($17, 0), $1, null:VARCHAR)])
+      LogicalFilter(condition=[OR(=($2, 'New York'), =($2, 'Aarhus'))])
+        LogicalTableScan(table=[[druid, wikipedia]])
+
+!convertedPlan
+LogicalSort(sort0=[$0], dir0=[ASC])
+  LogicalAggregate(group=[{0}], cnt=[COUNT($1) FILTER $2], aall=[COUNT()])
+    LogicalProject(cityName=[$2], channel=[$1], $f3=[IS TRUE(>($17, 0))])
+      LogicalFilter(condition=[SEARCH($2, Sarg['Aarhus':VARCHAR(8), 'New York':VARCHAR(8)]:VARCHAR(8))])
+        LogicalTableScan(table=[[druid, wikipedia]])
+
+!logicalPlan
+DruidAggregate(group=[{0}], cnt=[COUNT($1) FILTER $2], aall=[COUNT()], druid=[logical])
+  DruidProject(cityName=[$2], channel=[$1], $f3=[IS TRUE(>($17, 0))], druid=[logical])
+    DruidFilter(condition=[SEARCH($2, Sarg['Aarhus':VARCHAR(8), 'New York':VARCHAR(8)]:VARCHAR(8))])
+      DruidTableScan(table=[[druid, wikipedia]], druid=[logical])
+
+!druidPlan
+{
+  "queryType" : "groupBy",
+  "dataSource" : {
+    "type" : "table",
+    "name" : "wikipedia"
+  },
+  "intervals" : {
+    "type" : "intervals",
+    "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ]
+  },
+  "filter" : {
+    "type" : "inType",
+    "column" : "cityName",
+    "matchValueType" : "STRING",
+    "sortedValues" : [ "Aarhus", "New York" ]
+  },
+  "granularity" : {
+    "type" : "all"
+  },
+  "dimensions" : [ {
+    "type" : "default",
+    "dimension" : "cityName",
+    "outputName" : "d0",
+    "outputType" : "STRING"
+  } ],
+  "aggregations" : [ {
+    "type" : "filtered",
+    "aggregator" : {
+      "type" : "count",
+      "name" : "a0"
+    },
+    "filter" : {
+      "type" : "and",
+      "fields" : [ {
+        "type" : "not",
+        "field" : {
+          "type" : "null",
+          "column" : "channel"
+        }
+      }, {
+        "type" : "range",
+        "column" : "delta",
+        "matchValueType" : "LONG",
+        "lower" : 0,
+        "lowerOpen" : true
+      } ]
+    },
+    "name" : "a0"
+  }, {
+    "type" : "count",
+    "name" : "a1"
+  } ],
+  "limitSpec" : {
+    "type" : "NoopLimitSpec"
+  }
+}
+!nativePlan
+

sql/src/main/java/org/apache/druid/sql/SqlToolbox.java

@@ -51,6 +51,8 @@ public SqlToolbox(
   )
   {
     this.engine = engine;
+    System.out.println("SqlToolbox.SqlToolbox");
+    System.out.println("engine = " + engine);
     this.plannerFactory = plannerFactory;
     this.emitter = emitter;
     this.requestLogger = requestLogger;

sql/src/main/java/org/apache/druid/sql/calcite/planner/QueryHandler.java

@@ -746,6 +746,7 @@ protected PlannerResult planForDruid() throws ValidationException
     @Override
     protected QueryMaker buildQueryMaker(final RelRoot rootQueryRel) throws ValidationException
     {
+      System.out.println("SelectHandler.buildQueryMaker");
       return handlerContext.engine().buildQueryMakerForSelect(
           rootQueryRel,
           handlerContext.plannerContext()

sql/src/test/java/org/apache/druid/quidem/DruidQuidemTestBase.java

@@ -117,6 +117,7 @@ public void test(String testFileName) throws Exception
     File inFile = new File(getTestRoot(), testFileName);
 
     final File outFile = new File(inFile.getParentFile(), inFile.getName() + ".out");
+    System.out.println("druidQuidemRunner = " + druidQuidemRunner);
     druidQuidemRunner.run(inFile, outFile);
   }
 

sql/src/test/java/org/apache/druid/sql/calcite/SqlTestFrameworkConfig.java

@@ -470,6 +470,7 @@ private static Class<? extends QueryComponentSupplier> getQueryComponentSupplier
       Set<Class<? extends QueryComponentSupplier>> availableSuppliers = componentSupplierClassCache.get(pkg);
       for (Class<? extends QueryComponentSupplier> cl : availableSuppliers) {
         if (cl.getSimpleName().equals(name)) {
+          System.out.println("cl = " + cl);
           return cl;
         }
       }

sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java

@@ -628,6 +628,7 @@ private SqlTestFramework(Builder builder)
 
     this.injector = serviceInjector.build();
     this.engine = builder.componentSupplier.createEngine(queryLifecycleFactory(), queryJsonMapper(), injector);
+    System.out.println("engine = " + this.engine);
     componentSupplier.configureJsonMapper(queryJsonMapper());
     componentSupplier.finalizeTestFramework(this);
   }