Skip to content

Commit

Permalink
Update docker-images to 108
Browse files Browse the repository at this point in the history
  • Loading branch information
@ebyhr
ebyhr committed Jan 8, 2025
1 parent b6856b8 commit 3fffb1b
Show file tree
Hide file tree
Showing 4 changed files with 27 additions and 32 deletions.
16 changes: 9 additions & 7 deletions ...a-lake/src/main/java/io/trino/plugin/deltalake/transactionlog/DeltaLakeSchemaSupport.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -198,13 +198,15 @@ public static ColumnMappingMode getColumnMappingMode(MetadataEntry metadata, Pro
return ColumnMappingMode.NAME;
}

boolean supportsColumnMappingReader = protocolEntry.readerFeaturesContains(COLUMN_MAPPING_FEATURE_NAME);
boolean supportsColumnMappingWriter = protocolEntry.writerFeaturesContains(COLUMN_MAPPING_FEATURE_NAME);
checkArgument(
supportsColumnMappingReader == supportsColumnMappingWriter,
"Both reader and writer features must have the same value for 'columnMapping'. reader: %s, writer: %s", supportsColumnMappingReader, supportsColumnMappingWriter);
if (!supportsColumnMappingReader) {
return ColumnMappingMode.NONE;
if (protocolEntry.supportsReaderFeatures() && protocolEntry.supportsWriterFeatures()) {
boolean supportsColumnMappingReader = protocolEntry.readerFeaturesContains(COLUMN_MAPPING_FEATURE_NAME);
boolean supportsColumnMappingWriter = protocolEntry.writerFeaturesContains(COLUMN_MAPPING_FEATURE_NAME);
checkArgument(
supportsColumnMappingReader == supportsColumnMappingWriter,
"Both reader and writer features must have the same value for 'columnMapping'. reader: %s, writer: %s", supportsColumnMappingReader, supportsColumnMappingWriter);
if (!supportsColumnMappingReader) {
return ColumnMappingMode.NONE;
}
}
}
String columnMappingMode = metadata.getConfiguration().getOrDefault(COLUMN_MAPPING_MODE_CONFIGURATION_KEY, "none");
Expand Down
2 changes: 1 addition & 1 deletion pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -191,7 +191,7 @@
<dep.aws-sdk.version>1.12.780</dep.aws-sdk.version>
<dep.cassandra.version>4.17.0</dep.cassandra.version>
<dep.confluent.version>7.7.1</dep.confluent.version>
<dep.docker.images.version>107</dep.docker.images.version>
<dep.docker.images.version>108</dep.docker.images.version>
<dep.drift.version>1.22</dep.drift.version>
<dep.flyway.version>11.1.1</dep.flyway.version>
<dep.frontend-maven-plugin.version>1.15.1</dep.frontend-maven-plugin.version>
Expand Down
39 changes: 16 additions & 23 deletions ...-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeColumnMappingMode.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,6 @@
import static io.trino.tests.product.deltalake.util.DeltaLakeTestUtils.getTablePropertyOnDelta;
import static io.trino.tests.product.utils.QueryExecutors.onDelta;
import static io.trino.tests.product.utils.QueryExecutors.onTrino;
import static java.util.Locale.ENGLISH;
import static java.util.Objects.requireNonNull;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
Expand Down Expand Up @@ -106,20 +105,12 @@ public void testColumnMappingModeTableFeature(String mode)
.contains(entry("delta.feature.columnMapping", "supported"));

// Unsetting delta.columnMapping.mode means changing to 'none' column mapping mode
if (mode.equals("none")) {
onDelta().executeQuery("ALTER TABLE default." + tableName + " UNSET TBLPROPERTIES ('delta.columnMapping.mode')");
assertThat(getTablePropertiesOnDelta("default", tableName))
.contains(entry("delta.feature.columnMapping", "supported"))
.doesNotContainKey("delta.columnMapping.mode");
assertThat((String) onTrino().executeQuery("SHOW CREATE TABLE delta.default." + tableName).getOnlyValue())
.doesNotContain("column_mapping_mode =");
}
else {
assertQueryFailure(() -> onDelta().executeQuery("ALTER TABLE default." + tableName + " UNSET TBLPROPERTIES ('delta.columnMapping.mode')"))
.hasMessageContaining("Changing column mapping mode from '" + mode + "' to 'none' is not supported");
assertThat((String) onTrino().executeQuery("SHOW CREATE TABLE delta.default." + tableName).getOnlyValue())
.contains("column_mapping_mode = '" + mode.toUpperCase(ENGLISH) + "'");
}
onDelta().executeQuery("ALTER TABLE default." + tableName + " UNSET TBLPROPERTIES ('delta.columnMapping.mode')");
assertThat(getTablePropertiesOnDelta("default", tableName))
.contains(entry("delta.feature.columnMapping", "supported"))
.doesNotContainKey("delta.columnMapping.mode");
assertThat((String) onTrino().executeQuery("SHOW CREATE TABLE delta.default." + tableName).getOnlyValue())
.doesNotContain("column_mapping_mode =");
}
finally {
dropDeltaTableWithRetry("default." + tableName);
Expand All @@ -135,7 +126,8 @@ public void testTrinoColumnMappingModeReaderAndWriterVersion(String mode)
"WITH (" +
" location = 's3://" + bucketName + "/databricks-compatibility-test-" + tableName + "'," +
" column_mapping_mode = '" + mode + "'" +
")"));
")"),
5);
}

@Test(groups = {DELTA_LAKE_OSS, PROFILE_SPECIFIC_TESTS}, dataProvider = "columnMappingDataProvider")
Expand Down Expand Up @@ -196,7 +188,7 @@ public void testChangingColumnMappingModeViaCreateOrReplaceTableOnDelta(String m
"LOCATION 's3://" + bucketName + "/databricks-compatibility-test-" + tableName + "'" +
"TBLPROPERTIES ('delta.columnMapping.mode'='" + mode + "')");

assertTableReaderAndWriterVersion("default", tableName, "2", "5");
assertTableReaderAndWriterVersion("default", tableName, "2", "7");

// Revert back to `none` column mode
onDelta().executeQuery("" +
Expand All @@ -206,7 +198,7 @@ public void testChangingColumnMappingModeViaCreateOrReplaceTableOnDelta(String m
"LOCATION 's3://" + bucketName + "/databricks-compatibility-test-" + tableName + "'" +
"TBLPROPERTIES ('delta.columnMapping.mode'='none')");

assertTableReaderAndWriterVersion("default", tableName, "2", "5");
assertTableReaderAndWriterVersion("default", tableName, "2", "7");

onTrino().executeQuery("DROP TABLE delta.default." + tableName);
}
Expand All @@ -219,16 +211,17 @@ public void testDeltaColumnMappingModeReaderAndWriterVersion(String mode)
"(x INT) " +
"USING delta " +
"LOCATION 's3://" + bucketName + "/databricks-compatibility-test-" + tableName + "'" +
"TBLPROPERTIES ('delta.columnMapping.mode'='" + mode + "')"));
"TBLPROPERTIES ('delta.columnMapping.mode'='" + mode + "')"),
7);
}

private void testColumnMappingModeReaderAndWriterVersion(Consumer<String> createTable)
private void testColumnMappingModeReaderAndWriterVersion(Consumer<String> createTable, int expectedMinWriterVersion)
{
String tableName = "test_dl_column_mapping_version_" + randomNameSuffix();

createTable.accept(tableName);

assertTableReaderAndWriterVersion("default", tableName, "2", "5");
assertTableReaderAndWriterVersion("default", tableName, "2", Integer.toString(expectedMinWriterVersion));

onTrino().executeQuery("DROP TABLE delta.default." + tableName);
}
Expand Down Expand Up @@ -845,9 +838,9 @@ public Object[][] changeColumnMappingDataProvider()
// sourceMappingMode targetMappingMode supported
{"none", "id", false},
{"none", "name", true},
{"id", "none", false},
{"id", "none", true},
{"id", "name", false},
{"name", "none", false},
{"name", "none", true},
{"name", "id", false},
};
}
Expand Down
2 changes: 1 addition & 1 deletion ...src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeSystemTableCompatibility.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@ public void testTablePropertiesWithTableFeatures()
List<Row> expectedRows = ImmutableList.of(
row("delta.columnMapping.mode", "id"),
row("delta.feature.columnMapping", "supported"),
row("delta.minReaderVersion", "3"),
row("delta.minReaderVersion", "2"), // https://github.com/delta-io/delta/issues/4024 Delta Lake 3.3.0 ignores minReaderVersion
row("delta.minWriterVersion", "7"));
try {
QueryResult deltaResult = onDelta().executeQuery("SHOW TBLPROPERTIES default." + tableName);
Expand Down

0 comments on commit 3fffb1b

Please sign in to comment.