From 3fffb1b42d8a35a39b8e82b09195a691a6278d55 Mon Sep 17 00:00:00 2001 From: Yuya Ebihara Date: Tue, 7 Jan 2025 16:15:29 +0900 Subject: [PATCH] Update docker-images to 108 --- .../DeltaLakeSchemaSupport.java | 16 ++++---- pom.xml | 2 +- .../TestDeltaLakeColumnMappingMode.java | 39 ++++++++----------- ...TestDeltaLakeSystemTableCompatibility.java | 2 +- 4 files changed, 27 insertions(+), 32 deletions(-) diff --git a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/transactionlog/DeltaLakeSchemaSupport.java b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/transactionlog/DeltaLakeSchemaSupport.java index 67a73f8ad6a9..d9fc72df070a 100644 --- a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/transactionlog/DeltaLakeSchemaSupport.java +++ b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/transactionlog/DeltaLakeSchemaSupport.java @@ -198,13 +198,15 @@ public static ColumnMappingMode getColumnMappingMode(MetadataEntry metadata, Pro return ColumnMappingMode.NAME; } - boolean supportsColumnMappingReader = protocolEntry.readerFeaturesContains(COLUMN_MAPPING_FEATURE_NAME); - boolean supportsColumnMappingWriter = protocolEntry.writerFeaturesContains(COLUMN_MAPPING_FEATURE_NAME); - checkArgument( - supportsColumnMappingReader == supportsColumnMappingWriter, - "Both reader and writer features must have the same value for 'columnMapping'. reader: %s, writer: %s", supportsColumnMappingReader, supportsColumnMappingWriter); - if (!supportsColumnMappingReader) { - return ColumnMappingMode.NONE; + if (protocolEntry.supportsReaderFeatures() && protocolEntry.supportsWriterFeatures()) { + boolean supportsColumnMappingReader = protocolEntry.readerFeaturesContains(COLUMN_MAPPING_FEATURE_NAME); + boolean supportsColumnMappingWriter = protocolEntry.writerFeaturesContains(COLUMN_MAPPING_FEATURE_NAME); + checkArgument( + supportsColumnMappingReader == supportsColumnMappingWriter, + "Both reader and writer features must have the same value for 'columnMapping'. reader: %s, writer: %s", supportsColumnMappingReader, supportsColumnMappingWriter); + if (!supportsColumnMappingReader) { + return ColumnMappingMode.NONE; + } } } String columnMappingMode = metadata.getConfiguration().getOrDefault(COLUMN_MAPPING_MODE_CONFIGURATION_KEY, "none"); diff --git a/pom.xml b/pom.xml index 9f8bdb584582..a9ac466c0d75 100644 --- a/pom.xml +++ b/pom.xml @@ -191,7 +191,7 @@ 1.12.780 4.17.0 7.7.1 - 107 + 108 1.22 11.1.1 1.15.1 diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeColumnMappingMode.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeColumnMappingMode.java index 965eda2d4df6..9037f8c833a9 100644 --- a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeColumnMappingMode.java +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeColumnMappingMode.java @@ -45,7 +45,6 @@ import static io.trino.tests.product.deltalake.util.DeltaLakeTestUtils.getTablePropertyOnDelta; import static io.trino.tests.product.utils.QueryExecutors.onDelta; import static io.trino.tests.product.utils.QueryExecutors.onTrino; -import static java.util.Locale.ENGLISH; import static java.util.Objects.requireNonNull; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -106,20 +105,12 @@ public void testColumnMappingModeTableFeature(String mode) .contains(entry("delta.feature.columnMapping", "supported")); // Unsetting delta.columnMapping.mode means changing to 'none' column mapping mode - if (mode.equals("none")) { - onDelta().executeQuery("ALTER TABLE default." + tableName + " UNSET TBLPROPERTIES ('delta.columnMapping.mode')"); - assertThat(getTablePropertiesOnDelta("default", tableName)) - .contains(entry("delta.feature.columnMapping", "supported")) - .doesNotContainKey("delta.columnMapping.mode"); - assertThat((String) onTrino().executeQuery("SHOW CREATE TABLE delta.default." + tableName).getOnlyValue()) - .doesNotContain("column_mapping_mode ="); - } - else { - assertQueryFailure(() -> onDelta().executeQuery("ALTER TABLE default." + tableName + " UNSET TBLPROPERTIES ('delta.columnMapping.mode')")) - .hasMessageContaining("Changing column mapping mode from '" + mode + "' to 'none' is not supported"); - assertThat((String) onTrino().executeQuery("SHOW CREATE TABLE delta.default." + tableName).getOnlyValue()) - .contains("column_mapping_mode = '" + mode.toUpperCase(ENGLISH) + "'"); - } + onDelta().executeQuery("ALTER TABLE default." + tableName + " UNSET TBLPROPERTIES ('delta.columnMapping.mode')"); + assertThat(getTablePropertiesOnDelta("default", tableName)) + .contains(entry("delta.feature.columnMapping", "supported")) + .doesNotContainKey("delta.columnMapping.mode"); + assertThat((String) onTrino().executeQuery("SHOW CREATE TABLE delta.default." + tableName).getOnlyValue()) + .doesNotContain("column_mapping_mode ="); } finally { dropDeltaTableWithRetry("default." + tableName); @@ -135,7 +126,8 @@ public void testTrinoColumnMappingModeReaderAndWriterVersion(String mode) "WITH (" + " location = 's3://" + bucketName + "/databricks-compatibility-test-" + tableName + "'," + " column_mapping_mode = '" + mode + "'" + - ")")); + ")"), + 5); } @Test(groups = {DELTA_LAKE_OSS, PROFILE_SPECIFIC_TESTS}, dataProvider = "columnMappingDataProvider") @@ -196,7 +188,7 @@ public void testChangingColumnMappingModeViaCreateOrReplaceTableOnDelta(String m "LOCATION 's3://" + bucketName + "/databricks-compatibility-test-" + tableName + "'" + "TBLPROPERTIES ('delta.columnMapping.mode'='" + mode + "')"); - assertTableReaderAndWriterVersion("default", tableName, "2", "5"); + assertTableReaderAndWriterVersion("default", tableName, "2", "7"); // Revert back to `none` column mode onDelta().executeQuery("" + @@ -206,7 +198,7 @@ public void testChangingColumnMappingModeViaCreateOrReplaceTableOnDelta(String m "LOCATION 's3://" + bucketName + "/databricks-compatibility-test-" + tableName + "'" + "TBLPROPERTIES ('delta.columnMapping.mode'='none')"); - assertTableReaderAndWriterVersion("default", tableName, "2", "5"); + assertTableReaderAndWriterVersion("default", tableName, "2", "7"); onTrino().executeQuery("DROP TABLE delta.default." + tableName); } @@ -219,16 +211,17 @@ public void testDeltaColumnMappingModeReaderAndWriterVersion(String mode) "(x INT) " + "USING delta " + "LOCATION 's3://" + bucketName + "/databricks-compatibility-test-" + tableName + "'" + - "TBLPROPERTIES ('delta.columnMapping.mode'='" + mode + "')")); + "TBLPROPERTIES ('delta.columnMapping.mode'='" + mode + "')"), + 7); } - private void testColumnMappingModeReaderAndWriterVersion(Consumer createTable) + private void testColumnMappingModeReaderAndWriterVersion(Consumer createTable, int expectedMinWriterVersion) { String tableName = "test_dl_column_mapping_version_" + randomNameSuffix(); createTable.accept(tableName); - assertTableReaderAndWriterVersion("default", tableName, "2", "5"); + assertTableReaderAndWriterVersion("default", tableName, "2", Integer.toString(expectedMinWriterVersion)); onTrino().executeQuery("DROP TABLE delta.default." + tableName); } @@ -845,9 +838,9 @@ public Object[][] changeColumnMappingDataProvider() // sourceMappingMode targetMappingMode supported {"none", "id", false}, {"none", "name", true}, - {"id", "none", false}, + {"id", "none", true}, {"id", "name", false}, - {"name", "none", false}, + {"name", "none", true}, {"name", "id", false}, }; } diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeSystemTableCompatibility.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeSystemTableCompatibility.java index 63417308ebb7..842737d9ed3c 100644 --- a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeSystemTableCompatibility.java +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeSystemTableCompatibility.java @@ -79,7 +79,7 @@ public void testTablePropertiesWithTableFeatures() List expectedRows = ImmutableList.of( row("delta.columnMapping.mode", "id"), row("delta.feature.columnMapping", "supported"), - row("delta.minReaderVersion", "3"), + row("delta.minReaderVersion", "2"), // https://github.com/delta-io/delta/issues/4024 Delta Lake 3.3.0 ignores minReaderVersion row("delta.minWriterVersion", "7")); try { QueryResult deltaResult = onDelta().executeQuery("SHOW TBLPROPERTIES default." + tableName);