SQL: Add examples fo muting sql/csv integ tests (#47291)

matriv · web-flow · commit 591bba46516d · 2019-10-01T09:11:34.000+03:00
Add examples of failures for both sql and csv integeration
tests and instructions on how to mute them.
diff --git a/x-pack/plugin/sql/qa/src/main/resources/agg-ordering.sql-spec b/x-pack/plugin/sql/qa/src/main/resources/agg-ordering.sql-spec
@@ -1,3 +1,5 @@
+// To mute tests follow example in file: example.sql-spec
+
 //
 // Custom sorting/ordering on aggregates
 //
diff --git a/x-pack/plugin/sql/qa/src/main/resources/agg.csv-spec b/x-pack/plugin/sql/qa/src/main/resources/agg.csv-spec
@@ -1,3 +1,5 @@
+// To mute tests follow example in file: example.csv-spec
+
 //
 // Aggs not supported by H2 / traditional SQL stores
 //
diff --git a/x-pack/plugin/sql/qa/src/main/resources/agg.sql-spec b/x-pack/plugin/sql/qa/src/main/resources/agg.sql-spec
@@ -1,3 +1,5 @@
+// To mute tests follow example in file: example.sql-spec
+
 //
 // Group-By
 //
diff --git a/x-pack/plugin/sql/qa/src/main/resources/alias.csv-spec b/x-pack/plugin/sql/qa/src/main/resources/alias.csv-spec
@@ -1,3 +1,5 @@
+// To mute tests follow example in file: example.csv-spec
+
 tableAlias
 SELECT emp_no, first_name FROM test_alias ORDER BY emp_no LIMIT 6;
 
diff --git a/x-pack/plugin/sql/qa/src/main/resources/arithmetic.csv-spec b/x-pack/plugin/sql/qa/src/main/resources/arithmetic.csv-spec
@@ -1,3 +1,5 @@
+// To mute tests follow example in file: example.csv-spec
+
 //
 // Arithmetic tests outside H2
 //
diff --git a/x-pack/plugin/sql/qa/src/main/resources/arithmetic.sql-spec b/x-pack/plugin/sql/qa/src/main/resources/arithmetic.sql-spec
@@ -1,3 +1,5 @@
+// To mute tests follow example in file: example.sql-spec
+
 //
 // Arithmetic tests
 //
diff --git a/x-pack/plugin/sql/qa/src/main/resources/case-functions.sql-spec b/x-pack/plugin/sql/qa/src/main/resources/case-functions.sql-spec
@@ -1,3 +1,5 @@
+// To mute tests follow example in file: example.sql-spec
+
 // Next 4 SELECTs in this file are related to https://github.com/elastic/elasticsearch/issues/32589
 // H2 is Locale sensitive, while ES-SQL is not (so far)
 selectInsertWithLcaseAndLengthWithOrderBy
diff --git a/x-pack/plugin/sql/qa/src/main/resources/columns.csv-spec b/x-pack/plugin/sql/qa/src/main/resources/columns.csv-spec
@@ -1,3 +1,5 @@
+// To mute tests follow example in file: example.csv-spec
+
 //
 // Test of explicit column types
 // the columns can be specified as <ColumnName:ColumnType> or as <ColumnName>
@@ -13,4 +15,4 @@ gender:s             | p1:l
 M                    | 10096
 F                    | 10099
 null                 | 10019
-;
+;
diff --git a/x-pack/plugin/sql/qa/src/main/resources/command.csv-spec b/x-pack/plugin/sql/qa/src/main/resources/command.csv-spec
@@ -1,3 +1,5 @@
+// To mute tests follow example in file: example.csv-spec
+
 //
 // Commands
 //
diff --git a/x-pack/plugin/sql/qa/src/main/resources/conditionals.csv-spec b/x-pack/plugin/sql/qa/src/main/resources/conditionals.csv-spec
@@ -1,3 +1,5 @@
+// To mute tests follow example in file: example.csv-spec
+
 caseField
 SELECT emp_no, CASE WHEN emp_no - 10000 < 10 THEN 'First 10' ELSE 'Second 10' END as "case" FROM test_emp WHERE emp_no >= 10005
 ORDER BY emp_no LIMIT 10;
diff --git a/x-pack/plugin/sql/qa/src/main/resources/date.csv-spec b/x-pack/plugin/sql/qa/src/main/resources/date.csv-spec
@@ -1,3 +1,5 @@
+// To mute tests follow example in file: example.csv-spec
+
 //
 // Date
 //
diff --git a/x-pack/plugin/sql/qa/src/main/resources/datetime-interval.csv-spec b/x-pack/plugin/sql/qa/src/main/resources/datetime-interval.csv-spec
@@ -1,3 +1,5 @@
+// To mute tests follow example in file: example.csv-spec
+
 //
 // Unfortunately H2 doesn't support INTERVAL type,
 // hence why all INTERVAL tests need to be done manually
@@ -402,4 +404,4 @@ SELECT DOM(birth_date + INTERVAL 5 YEAR) x, COUNT(*) c FROM test_emp GROUP BY x
 7              |5              
 1              |5              
 null           |10             
-;
+;
diff --git a/x-pack/plugin/sql/qa/src/main/resources/datetime.csv-spec b/x-pack/plugin/sql/qa/src/main/resources/datetime.csv-spec
@@ -1,3 +1,5 @@
+// To mute tests follow example in file: example.csv-spec
+
 //
 // DateTime
 //
diff --git a/x-pack/plugin/sql/qa/src/main/resources/datetime.sql-spec b/x-pack/plugin/sql/qa/src/main/resources/datetime.sql-spec
@@ -1,3 +1,5 @@
+// To mute tests follow example in file: example.sql-spec
+
 //
 // DateTime
 //
diff --git a/x-pack/plugin/sql/qa/src/main/resources/docs/docs.csv-spec b/x-pack/plugin/sql/qa/src/main/resources/docs/docs.csv-spec
@@ -1,3 +1,5 @@
+// To mute tests follow example in file: example.csv-spec
+
 //
 // CSV spec used by the docs
 //
diff --git a/x-pack/plugin/sql/qa/src/main/resources/docs/geo.csv-spec b/x-pack/plugin/sql/qa/src/main/resources/docs/geo.csv-spec
@@ -1,3 +1,5 @@
+// To mute tests follow example in file: example.csv-spec
+
 //
 // CSV spec used by the geo docs
 //
diff --git a/x-pack/plugin/sql/qa/src/main/resources/examples/example.csv-spec b/x-pack/plugin/sql/qa/src/main/resources/examples/example.csv-spec
@@ -23,4 +23,53 @@ foo,2.5,3
 bar,3.5,4
 tar,4.5,5
 ;
-// repeat the above
+// repeat the above
+
+
+// Test failure example:
+// org.elasticsearch.xpack.sql.qa.single_node.JdbcCsvSpecIT > test {pivot.testAverageWithOneValueAndLimit} FAILED
+//    java.lang.AssertionError: Elasticsearch [JdbcResultSet:row 4:cursor size 2:[languages<type=[BYTE]>, 'F'<type=[DOUBLE]>]] still has data after [3] entries:
+//    3                    | 53660.0
+//        at __randomizedtesting.SeedInfo.seed([81AF0740AC9F7D95:9FB389A0263106D]:0)
+//        at org.elasticsearch.xpack.sql.qa.single_node.JdbcCsvSpecIT.pivot.testAverageWithOneValueAndLimit(pivot.csv-spec:95)
+// REPRODUCE WITH: ./gradlew ':x-pack:plugin:sql:qa:single-node:integTestRunner' --tests "org.elasticsearch.xpack.sql.qa.single_node.JdbcCsvSpecIT" -Dtests.method="test {pivot.testAverageWithOneValueAndLimit}" -Dtests.seed=81AF0740AC9F7D95 -Dtests.security.manager=true -Dtests.locale=yo -Dtests.timezone=Etc/GMT-10 -Dcompiler.java=12 -Druntime.java=11
+
+// Muting test example:
+// The name of the file which contains the failing test can be found by 2 components:
+// - JdbcCsvSpecIT shows that it's a .csv-spec suffixed file
+// - The name of the file is the part before the . so in our case: {pivot.testAverageWithOneValueAndLimit} => pivot
+// Which in our example gives: pivot.csv-spec
+//
+// You can also find the whole name in the stacktrace of the exception along with the line where the failed test resides:
+//    at org.elasticsearch.xpack.sql.qa.single_node.JdbcCsvSpecIT.pivot.testAverageWithOneValueAndLimit(pivot.csv-spec:95)
+//
+//
+// BEFORE (file: pivot.csv-spec):
+//----------------------------------------------
+// averageWithOneValueAndLimit
+// schema::languages:bt|'F':d
+// SELECT * FROM (SELECT languages, gender, salary FROM test_emp) PIVOT (AVG(salary) FOR gender IN ('F')) LIMIT 3;
+//
+//    languages   |       'F'
+// ---------------+------------------
+// null           |62140.666666666664
+// 1              |47073.25
+// 2              |50684.4
+// ;
+//----------------------------------------------
+//
+//
+// AFTER (muted), Just add the suffix -Ignore to the test name and a comment pointing to the Github issue:
+//----------------------------------------------
+// // AwaitsFix https://github.com/elastic/elasticsearch/issues/47080
+// averageWithOneValueAndLimit-Ignore
+// schema::languages:bt|'F':d
+// SELECT * FROM (SELECT languages, gender, salary FROM test_emp) PIVOT (AVG(salary) FOR gender IN ('F')) LIMIT 3;
+//
+//    languages   |       'F'
+// ---------------+------------------
+// null           |62140.666666666664
+// 1              |47073.25
+// 2              |50684.4
+// ;
+// ---------------------------------------------
diff --git a/x-pack/plugin/sql/qa/src/main/resources/examples/example.sql-spec b/x-pack/plugin/sql/qa/src/main/resources/examples/example.sql-spec
@@ -5,4 +5,36 @@ name
 // SQL query to be executed against H2 and ES
 SELECT COUNT(*) FROM "emp";
 
-// repeat the above
+// repeat the above
+
+
+// Test failure example:
+// org.elasticsearch.xpack.sql.qa.security.JdbcSqlSpecIT > test {agg.testGroupByOnText} FAILED
+//     org.h2.jdbc.JdbcSQLException: Column "gender2" not found; SQL statement:
+//     SELECT gender g FROM "test_emp" GROUP BY gender2 ORDER BY gender ASC [42122-197]
+//         at __randomizedtesting.SeedInfo.seed([880F4E292A732941:5B71F3848F44B9]:0)
+//         at org.elasticsearch.xpack.sql.qa.security.JdbcSqlSpecIT.agg.testGroupByOnText(agg.sql-spec:11)
+// REPRODUCE WITH: ./gradlew ':x-pack:plugin:sql:qa:security:without-ssl:integTestRunner' --tests "org.elasticsearch.xpack.sql.qa.security.JdbcSqlSpecIT" -Dtests.method="test {agg.testGroupByOnText}" -Dtests.seed=BB1082EEC0023E5C -Dtests.security.manager=true -Dtests.locale=pt-TL -Dtests.timezone=MST7MDT -Dcompiler.java=12 -Druntime.java=12
+
+// Muting test example:
+// The name of the file which contains the failing test can be found by 2 components:
+// - JdbcSqlSpecIT shows that it's a .sql-spec suffixed file
+// - The name of the file is the part before the . so in our case: agg.testGroupByOnText => agg
+// Which in our example gives: agg.sql-spec
+//
+// You can also find the whole name in the stacktrace of the exception along with the line where the failed test resides:
+//    at org.elasticsearch.xpack.sql.qa.security.JdbcSqlSpecIT.agg.testGroupByOnText(agg.sql-spec:11)
+//
+// BEFORE (file: agg.sql-spec):
+//----------------------------------------------
+// groupByOnText
+// SELECT gender g FROM "test_emp" GROUP BY gender ORDER BY gender ASC;
+//----------------------------------------------
+//
+//
+// AFTER (muted), Just add the suffix -Ignore to the test name and a comment pointing to the Github issue:
+//----------------------------------------------
+// // AwaitsFix https://github.com/elastic/elasticsearch/issues/33519
+// groupByOnText-Ignore
+// SELECT gender g FROM "test_emp" GROUP BY gender ORDER BY gender ASC;
+// ---------------------------------------------
diff --git a/x-pack/plugin/sql/qa/src/main/resources/field-alias.csv-spec b/x-pack/plugin/sql/qa/src/main/resources/field-alias.csv-spec
@@ -1,3 +1,5 @@
+// To mute tests follow example in file: example.csv-spec
+
 //
 // Tests testing field alias (introduced in ES 6.4)
 //
diff --git a/x-pack/plugin/sql/qa/src/main/resources/filter.csv-spec b/x-pack/plugin/sql/qa/src/main/resources/filter.csv-spec
@@ -1,3 +1,5 @@
+// To mute tests follow example in file: example.csv-spec
+
 //
 // Filter
 //
diff --git a/x-pack/plugin/sql/qa/src/main/resources/filter.sql-spec b/x-pack/plugin/sql/qa/src/main/resources/filter.sql-spec
@@ -1,3 +1,5 @@
+// To mute tests follow example in file: example.sql-spec
+
 //
 // Filter
 //
diff --git a/x-pack/plugin/sql/qa/src/main/resources/fulltext.csv-spec b/x-pack/plugin/sql/qa/src/main/resources/fulltext.csv-spec
@@ -1,3 +1,5 @@
+// To mute tests follow example in file: example.csv-spec
+
 //
 // Full-text
 //
diff --git a/x-pack/plugin/sql/qa/src/main/resources/functions.csv-spec b/x-pack/plugin/sql/qa/src/main/resources/functions.csv-spec
@@ -1,3 +1,5 @@
+// To mute tests follow example in file: example.csv-spec
+
 bitLengthGroupByAndOrderBy
 SELECT BIT_LENGTH(first_name), COUNT(*) count FROM "test_emp" GROUP BY BIT_LENGTH(first_name) ORDER BY BIT_LENGTH(first_name) LIMIT 10;
 
diff --git a/x-pack/plugin/sql/qa/src/main/resources/geo/geosql.csv-spec b/x-pack/plugin/sql/qa/src/main/resources/geo/geosql.csv-spec
@@ -1,3 +1,5 @@
+// To mute tests follow example in file: example.csv-spec
+
 //
 // Commands on geo test data
 //
@@ -301,4 +303,4 @@ SELECT ST_Y(ST_WKTToSQL('LINESTRING (200 100, 300 400)')) y;
       y:d
 100.0
 // end::y
-;
+;
diff --git a/x-pack/plugin/sql/qa/src/main/resources/geo/geosql.sql-spec b/x-pack/plugin/sql/qa/src/main/resources/geo/geosql.sql-spec
@@ -1,3 +1,5 @@
+// To mute tests follow example in file: example.sql-spec
+
 //
 // Commands on geo test data
 //
diff --git a/x-pack/plugin/sql/qa/src/main/resources/ip.csv-spec b/x-pack/plugin/sql/qa/src/main/resources/ip.csv-spec
@@ -1,3 +1,5 @@
+// To mute tests follow example in file: example.csv-spec
+
 //
 // Tests for IP fields
 //
diff --git a/x-pack/plugin/sql/qa/src/main/resources/math.csv-spec b/x-pack/plugin/sql/qa/src/main/resources/math.csv-spec
@@ -1,3 +1,5 @@
+// To mute tests follow example in file: example.csv-spec
+
 // this one doesn't work in H2 at all
 truncateWithAsciiHavingAndOrderBy
 SELECT TRUNCATE(ASCII(LEFT(first_name, 1)), 1), COUNT(*) count FROM test_emp GROUP BY ASCII(LEFT(first_name, 1)) HAVING COUNT(*) > 5 ORDER BY TRUNCATE(ASCII(LEFT(first_name, 1)), 1) DESC;
diff --git a/x-pack/plugin/sql/qa/src/main/resources/math.sql-spec b/x-pack/plugin/sql/qa/src/main/resources/math.sql-spec
@@ -1,3 +1,5 @@
+// To mute tests follow example in file: example.sql-spec
+
 //
 // Math
 //
diff --git a/x-pack/plugin/sql/qa/src/main/resources/nested.csv-spec b/x-pack/plugin/sql/qa/src/main/resources/nested.csv-spec
@@ -1,3 +1,5 @@
+// To mute tests follow example in file: example.csv-spec
+
 //
 //  Nested documents
 //
diff --git a/x-pack/plugin/sql/qa/src/main/resources/null.csv-spec b/x-pack/plugin/sql/qa/src/main/resources/null.csv-spec
@@ -1,3 +1,5 @@
+// To mute tests follow example in file: example.csv-spec
+
 //
 // Null expressions
 //
diff --git a/x-pack/plugin/sql/qa/src/main/resources/null.sql-spec b/x-pack/plugin/sql/qa/src/main/resources/null.sql-spec
@@ -1,3 +1,5 @@
+// To mute tests follow example in file: example.sql-spec
+
 //
 // Null conditionals
 //
diff --git a/x-pack/plugin/sql/qa/src/main/resources/ogc/ogc.csv-spec b/x-pack/plugin/sql/qa/src/main/resources/ogc/ogc.csv-spec
@@ -1,3 +1,5 @@
+// To mute tests follow example in file: example.csv-spec
+
 //
 // Commands on OGC data
 //
diff --git a/x-pack/plugin/sql/qa/src/main/resources/ogc/ogc.sql-spec b/x-pack/plugin/sql/qa/src/main/resources/ogc/ogc.sql-spec
@@ -1,3 +1,5 @@
+// To mute tests follow example in file: example.sql-spec
+
 //
 // Basic GEO SELECT
 //
diff --git a/x-pack/plugin/sql/qa/src/main/resources/pivot.csv-spec b/x-pack/plugin/sql/qa/src/main/resources/pivot.csv-spec
@@ -1,3 +1,5 @@
+// To mute tests follow example in file: example.csv-spec
+
 averageWithOneValue
 schema::languages:bt|'F':d
 SELECT * FROM (SELECT languages, gender, salary FROM test_emp) PIVOT (AVG(salary) FOR gender IN ('F'));
@@ -188,4 +190,4 @@ null           |31070.0
 3              |26830.0        
 4              |24646.0        
 5              |23353.0        
-;
+;
diff --git a/x-pack/plugin/sql/qa/src/main/resources/select.csv-spec b/x-pack/plugin/sql/qa/src/main/resources/select.csv-spec
@@ -1,3 +1,5 @@
+// To mute tests follow example in file: example.csv-spec
+
 //
 // SELECT with = and !=
 //
diff --git a/x-pack/plugin/sql/qa/src/main/resources/select.sql-spec b/x-pack/plugin/sql/qa/src/main/resources/select.sql-spec
@@ -1,3 +1,5 @@
+// To mute tests follow example in file: example.sql-spec
+
 //
 // Basic SELECT
 //
diff --git a/x-pack/plugin/sql/qa/src/main/resources/single-node-only/command-sys-geo.csv-spec b/x-pack/plugin/sql/qa/src/main/resources/single-node-only/command-sys-geo.csv-spec
@@ -1,3 +1,5 @@
+// To mute tests follow example in file: example.csv-spec
+
 //
 // Geo-specific Sys Commands
 //
@@ -13,4 +15,4 @@ integTest|null           |geo            |location_no_dv |114            |GEO_PO
 integTest|null           |geo            |region         |12             |KEYWORD        |32766          |2147483647     |null            |null            |1          |null           |null           |12             |0                 |2147483647         |5                 |YES          |null           |null          |null         |null               |NO                |NO
 integTest|null           |geo            |region_point   |12             |KEYWORD        |32766          |2147483647     |null            |null            |1          |null           |null           |12             |0                 |2147483647         |6                 |YES          |null           |null          |null         |null               |NO                |NO
 integTest|null           |geo            |shape          |114            |SHAPE          |2147483647     |2147483647     |null            |null            |1          |null           |null           |114            |0                 |null               |7                 |YES          |null           |null          |null         |null               |NO                |NO
-;
+;
diff --git a/x-pack/plugin/sql/qa/src/main/resources/single-node-only/command-sys.csv-spec b/x-pack/plugin/sql/qa/src/main/resources/single-node-only/command-sys.csv-spec
@@ -1,3 +1,5 @@
+// To mute tests follow example in file: example.csv-spec
+
 //
 // Sys Commands
 //
@@ -118,4 +120,4 @@ integTest  |null           |test_emp_copy  |languages         |-6             |B
 integTest  |null           |test_emp_copy  |last_name         |12             |TEXT           |2147483647     |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |14              |YES            |null           |null           |null           |null            |NO              |NO                
 integTest  |null           |test_emp_copy  |last_name.keyword |12             |KEYWORD        |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |15              |YES            |null           |null           |null           |null            |NO              |NO                
 integTest  |null           |test_emp_copy  |salary            |4              |INTEGER        |11             |4              |null           |10             |1              |null           |null           |4              |0               |null             |16              |YES            |null           |null           |null           |null            |NO              |NO                
-;
+;
diff --git a/x-pack/plugin/sql/qa/src/main/resources/slow/frozen.csv-spec b/x-pack/plugin/sql/qa/src/main/resources/slow/frozen.csv-spec
@@ -1,3 +1,5 @@
+// To mute tests follow example in file: example.csv-spec
+
 //
 // Frozen indices tests
 //
@@ -56,4 +58,4 @@ SELECT KURTOSIS(emp_no) k, SKEWNESS(salary) s FROM FROZEN frozen_emp;
 
 k:d                  | s:d
 1.7997599759975997   | 0.2707722118423227
-;
+;
diff --git a/x-pack/plugin/sql/qa/src/main/resources/string-functions.sql-spec b/x-pack/plugin/sql/qa/src/main/resources/string-functions.sql-spec
@@ -1,3 +1,5 @@
+// To mute tests follow example in file: example.sql-spec
+
 stringAscii
 SELECT ASCII(first_name) s FROM "test_emp" WHERE emp_no < 10010 ORDER BY emp_no;
 
diff --git a/x-pack/plugin/sql/qa/src/main/resources/time.csv-spec b/x-pack/plugin/sql/qa/src/main/resources/time.csv-spec
@@ -1,3 +1,5 @@
+// To mute tests follow example in file: example.csv-spec
+
 //
 // TIME
 //

Original file line number	Diff line number	Diff line change
`@@ -1,3 +1,5 @@`
	`1`	`+// To mute tests follow example in file: example.sql-spec`
	`2`	`+`
`1`	`3`	`//`
`2`	`4`	`// Custom sorting/ordering on aggregates`
`3`	`5`	`//`
Original file line number	Diff line number	Diff line change
`@@ -1,3 +1,5 @@`
	`1`	`+// To mute tests follow example in file: example.csv-spec`
	`2`	`+`
`1`	`3`	`tableAlias`
`2`	`4`	`SELECT emp_no, first_name FROM test_alias ORDER BY emp_no LIMIT 6;`
`3`	`5`
Original file line number	Diff line number	Diff line change
`@@ -1,3 +1,5 @@`
	`1`	`+// To mute tests follow example in file: example.sql-spec`
	`2`	`+`
`1`	`3`	`// Next 4 SELECTs in this file are related to https://github.com/elastic/elasticsearch/issues/32589`
`2`	`4`	`// H2 is Locale sensitive, while ES-SQL is not (so far)`
`3`	`5`	`selectInsertWithLcaseAndLengthWithOrderBy`
Original file line number	Diff line number	Diff line change
`@@ -1,3 +1,5 @@`
	`1`	`+// To mute tests follow example in file: example.csv-spec`
	`2`	`+`
`1`	`3`	`caseField`
`2`	`4`	`SELECT emp_no, CASE WHEN emp_no - 10000 < 10 THEN 'First 10' ELSE 'Second 10' END as "case" FROM test_emp WHERE emp_no >= 10005`
`3`	`5`	`ORDER BY emp_no LIMIT 10;`