fix(csharp/src/Drivers/BigQuery): update BigQuery test cases (#2048)

Fixes to make the following BigQuery tests pass: - VerifySchemaTablesWithNoConstraints - VerifyTypesAndValues --------- Co-authored-by: David Coe <coedavid@umich.edu>
apache · Aug 6, 2024 · 45cd9be · 45cd9be
1 parent d33b2d3
commit 45cd9be
Show file tree

Hide file tree

Showing 3 changed files with 112 additions and 94 deletions.
diff --git a/csharp/src/Apache.Arrow.Adbc/Extensions/StandardSchemaExtensions.cs b/csharp/src/Apache.Arrow.Adbc/Extensions/StandardSchemaExtensions.cs
@@ -53,6 +53,7 @@ public static IReadOnlyList<IArrowArray> Validate(this IReadOnlyList<Field> sche
  {
  Field field = schemaFields[i];
  ArrayData dataField = data[i].Data;
+
  if (field.DataType.TypeId != dataField.DataType.TypeId)
  {
  throw new ArgumentException($"Expecting data type {field.DataType} but found {data[i].Data.DataType} on field with name {field.Name}.", nameof(data));
@@ -65,10 +66,25 @@ public static IReadOnlyList<IArrowArray> Validate(this IReadOnlyList<Field> sche
  else if (field.DataType.TypeId == ArrowTypeId.List)
  {
  ListType listType = (ListType)field.DataType;
- if (listType.Fields.Count > 0)
+ int j = 0;
+ Field f = listType.Fields[j];
+
+ List<Field> fieldsToValidate = new List<Field>();
+ List<ContainerArray> arrayDataToValidate = new List<ContainerArray>();
+
+ ArrayData? child = j < dataField.Children.Length ? dataField.Children[j] : null;
+
+ if (child != null)
  {
- Validate(listType.Fields, dataField.Children.Select(e => new ContainerArray(e)).ToList());
+ fieldsToValidate.Add(f);
+ arrayDataToValidate.Add(new ContainerArray(child));
  }
+ else if (!f.IsNullable)
+ {
+ throw new InvalidOperationException("Received a null value for a non-nullable field");
+ }
+
+ Validate(fieldsToValidate, arrayDataToValidate);
  }
  else if (field.DataType.TypeId == ArrowTypeId.Union)
  {

diff --git a/csharp/src/Drivers/BigQuery/BigQueryStatement.cs b/csharp/src/Drivers/BigQuery/BigQueryStatement.cs
@@ -261,10 +261,9 @@ private string SerializeToJson(StructArray structArray, int index)
  {
  List<Dictionary<string, object?>?> children = new List<Dictionary<string, object?>?>();
 
- if (structArray1.Length > 1)
+ for (int j = 0; j < structArray1.Length; j++)
  {
- for (int j = 0; j < structArray1.Length; j++)
- children.Add(ParseStructArray(structArray1, j));
+ children.Add(ParseStructArray(structArray1, j));
  }
 
  if (children.Count > 0)

diff --git a/csharp/test/Drivers/BigQuery/BigQueryData.cs b/csharp/test/Drivers/BigQuery/BigQueryData.cs
@@ -42,9 +42,11 @@ public static SampleDataBuilder GetSampleData()
 
  SampleDataBuilder sampleDataBuilder = new SampleDataBuilder();
 
+
  // standard values
  sampleDataBuilder.Samples.Add(
- new SampleData() {
+ new SampleData()
+ {
  Query = "SELECT " +
  "CAST(1 as INT64) as id, " +
  "CAST(1.23 as FLOAT64) as number, " +
@@ -63,25 +65,25 @@ public static SampleDataBuilder GetSampleData()
  "PARSE_JSON('{\"name\":\"Jane Doe\",\"age\":29}') as json",
  ExpectedValues = new List<ColumnNetTypeArrowTypeValue>()
  {
- new ColumnNetTypeArrowTypeValue("id", typeof(long), typeof(Int64Type), 1L),
- new ColumnNetTypeArrowTypeValue("number", typeof(double), typeof(DoubleType), 1.23d),
- new ColumnNetTypeArrowTypeValue("decimal", typeof(SqlDecimal), typeof(Decimal128Type), SqlDecimal.Parse("4.56")),
- new ColumnNetTypeArrowTypeValue("big_decimal", typeof(string), typeof(StringType), "7.89000000000000000000000000000000000000"),
- new ColumnNetTypeArrowTypeValue("is_active", typeof(bool), typeof(BooleanType), true),
- new ColumnNetTypeArrowTypeValue("name", typeof(string), typeof(StringType), "John Doe"),
- new ColumnNetTypeArrowTypeValue("data", typeof(byte[]), typeof(BinaryType), UTF8Encoding.UTF8.GetBytes("abc123")),
- new ColumnNetTypeArrowTypeValue("date", typeof(DateTime), typeof(Date64Type), new DateTime(2023, 9, 8)),
-#if NET6_0_OR_GREATER
- new ColumnNetTypeArrowTypeValue("time", typeof(TimeOnly), typeof(Time64Type), new TimeOnly(12, 34, 56)), //'12:34:56'
-#else
- new ColumnNetTypeArrowTypeValue("time", typeof(TimeSpan), typeof(Time64Type), new TimeSpan(12, 34, 56)),
-#endif
- new ColumnNetTypeArrowTypeValue("datetime", typeof(DateTimeOffset), typeof(TimestampType), new DateTimeOffset(new DateTime(2023, 9, 8, 12, 34, 56), TimeSpan.Zero)),
- new ColumnNetTypeArrowTypeValue("timestamp", typeof(DateTimeOffset), typeof(TimestampType), new DateTimeOffset(new DateTime(2023, 9, 8, 12, 34, 56), TimeSpan.Zero)),
- new ColumnNetTypeArrowTypeValue("point", typeof(string), typeof(StringType), "POINT(1 2)"),
- new ColumnNetTypeArrowTypeValue("numbers", typeof(Int64Array), typeof(ListType), numbersArray),
- new ColumnNetTypeArrowTypeValue("person", typeof(string), typeof(StringType), "{\"name\":\"John Doe\",\"age\":30}"),
- new ColumnNetTypeArrowTypeValue("json", typeof(string), typeof(StringType), "{\"age\":29,\"name\":\"Jane Doe\"}")
+  new ColumnNetTypeArrowTypeValue("id", typeof(long), typeof(Int64Type), 1L),
+  new ColumnNetTypeArrowTypeValue("number", typeof(double), typeof(DoubleType), 1.23d),
+  new ColumnNetTypeArrowTypeValue("decimal", typeof(SqlDecimal), typeof(Decimal128Type), SqlDecimal.Parse("4.56")),
+  new ColumnNetTypeArrowTypeValue("big_decimal", typeof(string), typeof(StringType), "7.89000000000000000000000000000000000000"),
+  new ColumnNetTypeArrowTypeValue("is_active", typeof(bool), typeof(BooleanType), true),
+  new ColumnNetTypeArrowTypeValue("name", typeof(string), typeof(StringType), "John Doe"),
+  new ColumnNetTypeArrowTypeValue("data", typeof(byte[]), typeof(BinaryType), UTF8Encoding.UTF8.GetBytes("abc123")),
+  new ColumnNetTypeArrowTypeValue("date", typeof(DateTime), typeof(Date64Type), new DateTime(2023, 9, 8)),
+ #if NET6_0_OR_GREATER
+  new ColumnNetTypeArrowTypeValue("time", typeof(TimeOnly), typeof(Time64Type), new TimeOnly(12, 34, 56)), //'12:34:56'
+ #else
+  new ColumnNetTypeArrowTypeValue("time", typeof(TimeSpan), typeof(Time64Type), new TimeSpan(12, 34, 56)),
+ #endif
+  new ColumnNetTypeArrowTypeValue("datetime", typeof(DateTimeOffset), typeof(TimestampType), new DateTimeOffset(new DateTime(2023, 9, 8, 12, 34, 56), TimeSpan.Zero)),
+  new ColumnNetTypeArrowTypeValue("timestamp", typeof(DateTimeOffset), typeof(TimestampType), new DateTimeOffset(new DateTime(2023, 9, 8, 12, 34, 56), TimeSpan.Zero)),
+  new ColumnNetTypeArrowTypeValue("point", typeof(string), typeof(StringType), "POINT(1 2)"),
+  new ColumnNetTypeArrowTypeValue("numbers", typeof(Int64Array), typeof(ListType), numbersArray),
+  new ColumnNetTypeArrowTypeValue("person", typeof(string), typeof(StringType), "{\"name\":\"John Doe\",\"age\":30}"),
+  new ColumnNetTypeArrowTypeValue("json", typeof(string), typeof(StringType), "{\"age\":29,\"name\":\"Jane Doe\"}")
  }
  });
 
@@ -125,82 +127,83 @@ public static SampleDataBuilder GetSampleData()
  "STRUCT(CAST(NULL as STRING) as name, CAST(NULL as INT64) as age) as person",
  ExpectedValues = new List<ColumnNetTypeArrowTypeValue>()
  {
- new ColumnNetTypeArrowTypeValue("id", typeof(long), typeof(Int64Type), null),
- new ColumnNetTypeArrowTypeValue("number", typeof(double), typeof(DoubleType), null),
- new ColumnNetTypeArrowTypeValue("decimal", typeof(SqlDecimal), typeof(Decimal128Type), null),
- new ColumnNetTypeArrowTypeValue("big_decimal", typeof(string), typeof(StringType), null),
- new ColumnNetTypeArrowTypeValue("is_active", typeof(bool), typeof(BooleanType), null),
- new ColumnNetTypeArrowTypeValue("name", typeof(string), typeof(StringType), null),
- new ColumnNetTypeArrowTypeValue("data", typeof(byte[]), typeof(BinaryType), null),
- new ColumnNetTypeArrowTypeValue("date", typeof(DateTime), typeof(Date64Type), null),
-#if NET6_0_OR_GREATER
- new ColumnNetTypeArrowTypeValue("time", typeof(TimeOnly), typeof(Time64Type), null),
-#else
- new ColumnNetTypeArrowTypeValue("time", typeof(TimeSpan), typeof(Time64Type), null),
-#endif
- new ColumnNetTypeArrowTypeValue("datetime", typeof(DateTimeOffset), typeof(TimestampType), null),
- new ColumnNetTypeArrowTypeValue("timestamp", typeof(DateTimeOffset), typeof(TimestampType), null),
- new ColumnNetTypeArrowTypeValue("point", typeof(string), typeof(StringType), null),
- new ColumnNetTypeArrowTypeValue("numbers", typeof(Int64Array), typeof(ListType), emptyNumbersArray),
- new ColumnNetTypeArrowTypeValue("person", typeof(string), typeof(StringType), "{\"name\":null,\"age\":null}")
+  new ColumnNetTypeArrowTypeValue("id", typeof(long), typeof(Int64Type), null),
+  new ColumnNetTypeArrowTypeValue("number", typeof(double), typeof(DoubleType), null),
+  new ColumnNetTypeArrowTypeValue("decimal", typeof(SqlDecimal), typeof(Decimal128Type), null),
+  new ColumnNetTypeArrowTypeValue("big_decimal", typeof(string), typeof(StringType), null),
+  new ColumnNetTypeArrowTypeValue("is_active", typeof(bool), typeof(BooleanType), null),
+  new ColumnNetTypeArrowTypeValue("name", typeof(string), typeof(StringType), null),
+  new ColumnNetTypeArrowTypeValue("data", typeof(byte[]), typeof(BinaryType), null),
+  new ColumnNetTypeArrowTypeValue("date", typeof(DateTime), typeof(Date64Type), null),
+ #if NET6_0_OR_GREATER
+  new ColumnNetTypeArrowTypeValue("time", typeof(TimeOnly), typeof(Time64Type), null),
+ #else
+  new ColumnNetTypeArrowTypeValue("time", typeof(TimeSpan), typeof(Time64Type), null),
+ #endif
+  new ColumnNetTypeArrowTypeValue("datetime", typeof(DateTimeOffset), typeof(TimestampType), null),
+  new ColumnNetTypeArrowTypeValue("timestamp", typeof(DateTimeOffset), typeof(TimestampType), null),
+  new ColumnNetTypeArrowTypeValue("point", typeof(string), typeof(StringType), null),
+  new ColumnNetTypeArrowTypeValue("numbers", typeof(Int64Array), typeof(ListType), emptyNumbersArray),
+  new ColumnNetTypeArrowTypeValue("person", typeof(string), typeof(StringType), "{\"name\":null,\"age\":null}")
  }
  });
 
+
  // complex struct
  sampleDataBuilder.Samples.Add(
-  new SampleData()
-  {
-  Query =  "SELECT " +
-  "STRUCT(" +
-  "\"Iron Man\" as name," +
-  "\"Avengers\" as team," +
-  "[\"Genius\", \"Billionaire\", \"Playboy\", \"Philanthropist\"] as powers," +
-  "[" +
-  " STRUCT(" +
-  " \"Captain America\" as name, " +
-  " \"Avengers\" as team, " +
-  " [\"Super Soldier Serum\", \"Vibranium Shield\"] as powers, " +
-  " [" +
-  " STRUCT(" +
-  " \"Thanos\" as name, " +
-  " \"Black Order\" as team, " +
-  " [\"Infinity Gauntlet\", \"Super Strength\", \"Teleportation\"] as powers, " +
-  " [" +
-  " STRUCT(" +
-  " \"Loki\" as name, " +
-  " \"Asgard\" as team, " +
-  " [\"Magic\", \"Shapeshifting\", \"Trickery\"] as powers " +
-  " )" +
-  " ] as allies" +
-  " )" +
-  " ] as enemies" +
-  " )," +
-  " STRUCT(" +
-  " \"Spider-Man\" as name, " +
-  " \"Avengers\" as team, " +
-  " [\"Spider-Sense\", \"Web-Shooting\", \"Wall-Crawling\"] as powers, " +
-  " [" +
-  " STRUCT(" +
-  " \"Green Goblin\" as name, " +
-  " \"Sinister Six\" as team, " +
-  " [\"Glider\", \"Pumpkin Bombs\", \"Super Strength\"] as powers, " +
-  " [" +
-  " STRUCT(" +
-  " \"Doctor Octopus\" as name, " +
-  " \"Sinister Six\" as team, " +
-  " [\"Mechanical Arms\", \"Genius\", \"Madness\"] as powers " +
-  " )" +
-  " ] as allies" +
-  " )" +
-  " ] as enemies" +
-  " )" +
-  " ] as friends" +
-  ") as iron_man",
-  ExpectedValues = new List<ColumnNetTypeArrowTypeValue>()
+ new SampleData()
+ {
+ Query = "SELECT " +
+ "STRUCT(" +
+ "\"Iron Man\" as name," +
+ "\"Avengers\" as team," +
+ "[\"Genius\", \"Billionaire\", \"Playboy\", \"Philanthropist\"] as powers," +
+ "[" +
+ " STRUCT(" +
+ " \"Captain America\" as name, " +
+ " \"Avengers\" as team, " +
+ " [\"Super Soldier Serum\", \"Vibranium Shield\"] as powers, " +
+ " [" +
+ " STRUCT(" +
+ " \"Thanos\" as name, " +
+ " \"Black Order\" as team, " +
+ " [\"Infinity Gauntlet\", \"Super Strength\", \"Teleportation\"] as powers, " +
+ " [" +
+ " STRUCT(" +
+ " \"Loki\" as name, " +
+ " \"Asgard\" as team, " +
+ " [\"Magic\", \"Shapeshifting\", \"Trickery\"] as powers " +
+ " )" +
+ " ] as allies" +
+ " )" +
+ " ] as enemies" +
+ " )," +
+ " STRUCT(" +
+ " \"Spider-Man\" as name, " +
+ " \"Avengers\" as team, " +
+ " [\"Spider-Sense\", \"Web-Shooting\", \"Wall-Crawling\"] as powers, " +
+ " [" +
+ " STRUCT(" +
+ " \"Green Goblin\" as name, " +
+ " \"Sinister Six\" as team, " +
+ " [\"Glider\", \"Pumpkin Bombs\", \"Super Strength\"] as powers, " +
+ " [" +
+ " STRUCT(" +
+ " \"Doctor Octopus\" as name, " +
+ " \"Sinister Six\" as team, " +
+ " [\"Mechanical Arms\", \"Genius\", \"Madness\"] as powers " +
+ " )" +
+ " ] as allies" +
+ " )" +
+ " ] as enemies" +
+ " )" +
+ " ] as friends" +
+ ") as iron_man",
+ ExpectedValues = new List<ColumnNetTypeArrowTypeValue>()
  {
- new ColumnNetTypeArrowTypeValue("iron_man", typeof(string), typeof(StringType), "{\"name\":\"Iron Man\",\"team\":\"Avengers\",\"powers\":[\"Genius\",\"Billionaire\",\"Playboy\",\"Philanthropist\"],\"friends\":[{\"name\":\"Captain America\",\"team\":\"Avengers\",\"powers\":[\"Super Soldier Serum\",\"Vibranium Shield\"],\"enemies\":{\"name\":\"Thanos\",\"team\":\"Black Order\",\"powers\":[\"Infinity Gauntlet\",\"Super Strength\",\"Teleportation\"],\"allies\":{\"name\":\"Loki\",\"team\":\"Asgard\",\"powers\":[\"Magic\",\"Shapeshifting\",\"Trickery\"]}}},{\"name\":\"Spider-Man\",\"team\":\"Avengers\",\"powers\":[\"Spider-Sense\",\"Web-Shooting\",\"Wall-Crawling\"],\"enemies\":{\"name\":\"Green Goblin\",\"team\":\"Sinister Six\",\"powers\":[\"Glider\",\"Pumpkin Bombs\",\"Super Strength\"],\"allies\":{\"name\":\"Doctor Octopus\",\"team\":\"Sinister Six\",\"powers\":[\"Mechanical Arms\",\"Genius\",\"Madness\"]}}}]}")
+ new ColumnNetTypeArrowTypeValue("iron_man", typeof(string), typeof(StringType), "{\"name\":\"Iron Man\",\"team\":\"Avengers\",\"powers\":[\"Genius\",\"Billionaire\",\"Playboy\",\"Philanthropist\"],\"friends\":[{\"name\":\"Captain America\",\"team\":\"Avengers\",\"powers\":[\"Super Soldier Serum\",\"Vibranium Shield\"],\"enemies\":[{\"name\":\"Thanos\",\"team\":\"Black Order\",\"powers\":[\"Infinity Gauntlet\",\"Super Strength\",\"Teleportation\"],\"allies\":[{\"name\":\"Loki\",\"team\":\"Asgard\",\"powers\":[\"Magic\",\"Shapeshifting\",\"Trickery\"]}]}]},{\"name\":\"Spider-Man\",\"team\":\"Avengers\",\"powers\":[\"Spider-Sense\",\"Web-Shooting\",\"Wall-Crawling\"],\"enemies\":[{\"name\":\"Green Goblin\",\"team\":\"Sinister Six\",\"powers\":[\"Glider\",\"Pumpkin Bombs\",\"Super Strength\"],\"allies\":[{\"name\":\"Doctor Octopus\",\"team\":\"Sinister Six\",\"powers\":[\"Mechanical Arms\",\"Genius\",\"Madness\"]}]}]}]}")
  }
-  });
+ });
 
  return sampleDataBuilder;
  }