-
Notifications
You must be signed in to change notification settings - Fork 3k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add support for typeWidening(-preview) reader feature in Delta Lake
- Loading branch information
Showing
36 changed files
with
258 additions
and
3 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
18 changes: 18 additions & 0 deletions
18
plugin/trino-delta-lake/src/test/resources/deltalake/type_widening/README.md
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,18 @@ | ||
Data generated using OSS Delta Lake 3.2.0: | ||
|
||
```sql | ||
CREATE TABLE default.test_type_widening | ||
(col byte) | ||
USING DELTA | ||
LOCATION 's3://test-bucket/databricks-compatibility-test-test-widening' | ||
TBLPROPERTIES ('delta.enableTypeWidening'=true); | ||
INSERT INTO default.test_type_widening VALUES 127; | ||
|
||
ALTER TABLE default.test_type_widening CHANGE COLUMN col TYPE short; | ||
INSERT INTO default.test_type_widening VALUES 32767; | ||
|
||
ALTER TABLE default.test_type_widening CHANGE COLUMN col TYPE integer; | ||
INSERT INTO default.test_type_widening VALUES 2147483647; | ||
``` | ||
|
||
Other type widening including from integer to long is not supported in 3.2.0. |
3 changes: 3 additions & 0 deletions
3
...elta-lake/src/test/resources/deltalake/type_widening/_delta_log/00000000000000000000.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,3 @@ | ||
{"commitInfo":{"timestamp":1716683236025,"operation":"CREATE TABLE","operationParameters":{"partitionBy":"[]","clusterBy":"[]","description":null,"isManaged":"false","properties":"{\"delta.enableTypeWidening\":\"true\"}"},"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.2.0","txnId":"ad489be2-2283-4cbf-8959-fe1ce69d9877"}} | ||
{"metaData":{"id":"3b653ec1-f12b-40c9-9e27-f05fa9941db3","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"col\",\"type\":\"byte\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{"delta.enableTypeWidening":"true"},"createdTime":1716683235880}} | ||
{"protocol":{"minReaderVersion":3,"minWriterVersion":7,"readerFeatures":["typeWidening-preview"],"writerFeatures":["typeWidening-preview"]}} |
2 changes: 2 additions & 0 deletions
2
...elta-lake/src/test/resources/deltalake/type_widening/_delta_log/00000000000000000001.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,2 @@ | ||
{"commitInfo":{"timestamp":1716683278791,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":0,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"1","numOutputRows":"1","numOutputBytes":"461"},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.2.0","txnId":"ba5336d4-ee05-4405-8c84-1694276e49b6"}} | ||
{"add":{"path":"part-00000-564b0b78-9796-45f9-8f3d-6f36fb01929b-c000.snappy.parquet","partitionValues":{},"size":461,"modificationTime":1716683278000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"col\":127},\"maxValues\":{\"col\":127},\"nullCount\":{\"col\":0}}","defaultRowCommitVersion":1}} |
2 changes: 2 additions & 0 deletions
2
...elta-lake/src/test/resources/deltalake/type_widening/_delta_log/00000000000000000002.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,2 @@ | ||
{"commitInfo":{"timestamp":1716683348167,"operation":"CHANGE COLUMN","operationParameters":{"column":"{\"name\":\"col\",\"type\":\"short\",\"nullable\":true,\"metadata\":{}}"},"readVersion":1,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.2.0","txnId":"9580b64d-9d56-4fef-bede-35cca56df178"}} | ||
{"metaData":{"id":"3b653ec1-f12b-40c9-9e27-f05fa9941db3","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"col\",\"type\":\"short\",\"nullable\":true,\"metadata\":{\"delta.typeChanges\":[{\"toType\":\"short\",\"fromType\":\"byte\",\"tableVersion\":2}]}}]}","partitionColumns":[],"configuration":{"delta.enableTypeWidening":"true"},"createdTime":1716683235880}} |
2 changes: 2 additions & 0 deletions
2
...elta-lake/src/test/resources/deltalake/type_widening/_delta_log/00000000000000000003.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,2 @@ | ||
{"commitInfo":{"timestamp":1716683349676,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":2,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"1","numOutputRows":"1","numOutputBytes":"538"},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.2.0","txnId":"f3405988-757d-4959-96d2-d7142d7162b6"}} | ||
{"add":{"path":"part-00000-e15d8e8c-2e34-4d76-bacf-6315428daba0-c000.snappy.parquet","partitionValues":{},"size":538,"modificationTime":1716683349000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"col\":32767},\"maxValues\":{\"col\":32767},\"nullCount\":{\"col\":0}}","defaultRowCommitVersion":3}} |
2 changes: 2 additions & 0 deletions
2
...elta-lake/src/test/resources/deltalake/type_widening/_delta_log/00000000000000000004.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,2 @@ | ||
{"commitInfo":{"timestamp":1716683377041,"operation":"CHANGE COLUMN","operationParameters":{"column":"{\"name\":\"col\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{\"delta.typeChanges\":[{\"toType\":\"short\",\"fromType\":\"byte\",\"tableVersion\":2}]}}"},"readVersion":3,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.2.0","txnId":"8f7e22df-f174-46a5-ba63-aa97c99a33cb"}} | ||
{"metaData":{"id":"3b653ec1-f12b-40c9-9e27-f05fa9941db3","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"col\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{\"delta.typeChanges\":[{\"toType\":\"short\",\"fromType\":\"byte\",\"tableVersion\":2},{\"toType\":\"integer\",\"fromType\":\"short\",\"tableVersion\":4}]}}]}","partitionColumns":[],"configuration":{"delta.enableTypeWidening":"true"},"createdTime":1716683235880}} |
2 changes: 2 additions & 0 deletions
2
...elta-lake/src/test/resources/deltalake/type_widening/_delta_log/00000000000000000005.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,2 @@ | ||
{"commitInfo":{"timestamp":1716683378212,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":4,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"1","numOutputRows":"1","numOutputBytes":"588"},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.2.0","txnId":"eabc76f4-236c-4113-8d8e-df1eface2ac3"}} | ||
{"add":{"path":"part-00000-9dc5a505-e7ae-4958-901d-f05b8f6a5209-c000.snappy.parquet","partitionValues":{},"size":588,"modificationTime":1716683378000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"col\":2147483647},\"maxValues\":{\"col\":2147483647},\"nullCount\":{\"col\":0}}","defaultRowCommitVersion":5}} |
Binary file added
BIN
+461 Bytes
...ltalake/type_widening/part-00000-564b0b78-9796-45f9-8f3d-6f36fb01929b-c000.snappy.parquet
Binary file not shown.
Binary file added
BIN
+588 Bytes
...ltalake/type_widening/part-00000-9dc5a505-e7ae-4958-901d-f05b8f6a5209-c000.snappy.parquet
Binary file not shown.
Binary file added
BIN
+538 Bytes
...ltalake/type_widening/part-00000-e15d8e8c-2e34-4d76-bacf-6315428daba0-c000.snappy.parquet
Binary file not shown.
24 changes: 24 additions & 0 deletions
24
...in/trino-delta-lake/src/test/resources/deltalake/type_widening_nested/README.md
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,24 @@ | ||
Data generated using OSS Delta Lake 3.2.0: | ||
|
||
```sql | ||
CREATE TABLE default.test_type_widening_nested | ||
(s struct<field: byte>, m map<byte, byte>, a array<byte>) | ||
USING DELTA | ||
LOCATION 's3://test-bucket/databricks-compatibility-test-test-widening-nested' | ||
TBLPROPERTIES ('delta.enableTypeWidening'=true); | ||
INSERT INTO default.test_type_widening_nested VALUES (named_struct('field',127), map(-128,127), array(127)); | ||
|
||
ALTER TABLE default.test_type_widening_nested CHANGE COLUMN s.field TYPE short; | ||
ALTER TABLE default.test_type_widening_nested CHANGE COLUMN m.key TYPE short; | ||
ALTER TABLE default.test_type_widening_nested CHANGE COLUMN m.value TYPE short; | ||
ALTER TABLE default.test_type_widening_nested CHANGE COLUMN a.element TYPE short; | ||
INSERT INTO default.test_type_widening_nested VALUES (named_struct('field',32767), map(-32768,32767), array(32767)); | ||
|
||
ALTER TABLE default.test_type_widening_nested CHANGE COLUMN s.field TYPE integer; | ||
ALTER TABLE default.test_type_widening_nested CHANGE COLUMN m.key TYPE integer; | ||
ALTER TABLE default.test_type_widening_nested CHANGE COLUMN m.value TYPE integer; | ||
ALTER TABLE default.test_type_widening_nested CHANGE COLUMN a.element TYPE integer; | ||
INSERT INTO default.test_type_widening_nested VALUES (named_struct('field',2147483647), map(-2147483648,2147483647), array(2147483647)); | ||
``` | ||
|
||
Other type widening including from integer to long is not supported in 3.2.0. |
3 changes: 3 additions & 0 deletions
3
...ke/src/test/resources/deltalake/type_widening_nested/_delta_log/00000000000000000000.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,3 @@ | ||
{"commitInfo":{"timestamp":1716686845777,"operation":"CREATE TABLE","operationParameters":{"partitionBy":"[]","clusterBy":"[]","description":null,"isManaged":"false","properties":"{\"delta.enableTypeWidening\":\"true\"}"},"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.2.0","txnId":"2ab59e14-0675-443b-86af-4a619fc97bab"}} | ||
{"metaData":{"id":"628f6682-9791-4ee0-a0c4-502239349046","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"s\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"field\",\"type\":\"byte\",\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}},{\"name\":\"m\",\"type\":{\"type\":\"map\",\"keyType\":\"byte\",\"valueType\":\"byte\",\"valueContainsNull\":true},\"nullable\":true,\"metadata\":{}},{\"name\":\"a\",\"type\":{\"type\":\"array\",\"elementType\":\"byte\",\"containsNull\":true},\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{"delta.enableTypeWidening":"true"},"createdTime":1716686845629}} | ||
{"protocol":{"minReaderVersion":3,"minWriterVersion":7,"readerFeatures":["typeWidening-preview"],"writerFeatures":["typeWidening-preview"]}} |
2 changes: 2 additions & 0 deletions
2
...ke/src/test/resources/deltalake/type_widening_nested/_delta_log/00000000000000000001.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,2 @@ | ||
{"commitInfo":{"timestamp":1716686867493,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":0,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"1","numOutputRows":"1","numOutputBytes":"1413"},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.2.0","txnId":"c162f32f-0e31-4967-943a-66787c2623b9"}} | ||
{"add":{"path":"part-00000-aa99b42e-b5ca-4faf-a9ff-cbcd951cc1b7-c000.snappy.parquet","partitionValues":{},"size":1413,"modificationTime":1716686867000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"s\":{\"field\":127}},\"maxValues\":{\"s\":{\"field\":127}},\"nullCount\":{\"s\":{\"field\":0},\"m\":0,\"a\":0}}","defaultRowCommitVersion":1}} |
2 changes: 2 additions & 0 deletions
2
...ke/src/test/resources/deltalake/type_widening_nested/_delta_log/00000000000000000002.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,2 @@ | ||
{"commitInfo":{"timestamp":1716686871922,"operation":"CHANGE COLUMN","operationParameters":{"column":"{\"name\":\"s.field\",\"type\":\"short\",\"nullable\":true,\"metadata\":{}}"},"readVersion":1,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.2.0","txnId":"265c814a-5bf9-41cd-9170-99b26b584b31"}} | ||
{"metaData":{"id":"628f6682-9791-4ee0-a0c4-502239349046","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"s\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"field\",\"type\":\"short\",\"nullable\":true,\"metadata\":{\"delta.typeChanges\":[{\"toType\":\"short\",\"fromType\":\"byte\",\"tableVersion\":2}]}}]},\"nullable\":true,\"metadata\":{}},{\"name\":\"m\",\"type\":{\"type\":\"map\",\"keyType\":\"byte\",\"valueType\":\"byte\",\"valueContainsNull\":true},\"nullable\":true,\"metadata\":{}},{\"name\":\"a\",\"type\":{\"type\":\"array\",\"elementType\":\"byte\",\"containsNull\":true},\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{"delta.enableTypeWidening":"true"},"createdTime":1716686845629}} |
Oops, something went wrong.