diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java index 28efbc58f51b84..5afef56f085c80 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java @@ -192,7 +192,11 @@ private void setPaimonParams(TFileRangeDesc rangeDesc, PaimonSplit paimonSplit) if (optDeletionFile.isPresent()) { DeletionFile deletionFile = optDeletionFile.get(); TPaimonDeletionFileDesc tDeletionFile = new TPaimonDeletionFileDesc(); - tDeletionFile.setPath(deletionFile.path()); + // convert the deletion file uri to make sure FileReader can read it in be + LocationPath locationPath = new LocationPath(deletionFile.path(), + source.getCatalog().getProperties()); + String path = locationPath.toStorageLocation().toString(); + tDeletionFile.setPath(path); tDeletionFile.setOffset(deletionFile.offset()); tDeletionFile.setLength(deletionFile.length()); fileDesc.setDeletionFile(tDeletionFile); diff --git a/regression-test/data/external_table_p0/paimon/test_paimon_deletion_vector_oss.out b/regression-test/data/external_table_p0/paimon/test_paimon_deletion_vector_oss.out new file mode 100644 index 00000000000000..f0b1e92a088538 --- /dev/null +++ b/regression-test/data/external_table_p0/paimon/test_paimon_deletion_vector_oss.out @@ -0,0 +1,73 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !1 -- +3 + +-- !2 -- +3 + +-- !3 -- +2 + +-- !4 -- +2 + +-- !5 -- +3 3_1 +4 4_1 + +-- !6 -- +3 3_1 +4 4_1 + +-- !7 -- +1 Paimon 5 +4 Venti 25 +5 Kaeya 28 +6 Jean 26 +7 Diluc 30 +8 Razor 18 +9 Mona 19 +10 Barbara 16 + +-- !8 -- +8 + +-- !9 -- +7 + +-- !1 -- +3 + +-- !2 -- +3 + +-- !3 -- +2 + +-- !4 -- +2 + +-- !5 -- +3 3_1 +4 4_1 + +-- !6 -- +3 3_1 +4 4_1 + +-- !7 -- +1 Paimon 5 +4 Venti 25 +5 Kaeya 28 +6 Jean 26 +7 Diluc 30 +8 Razor 18 +9 Mona 19 +10 Barbara 16 + +-- !8 -- +8 + +-- !9 -- +7 + diff --git a/regression-test/suites/external_table_p0/paimon/test_paimon_deletion_vector_oss.groovy b/regression-test/suites/external_table_p0/paimon/test_paimon_deletion_vector_oss.groovy new file mode 100644 index 00000000000000..55fc903a31b3a3 --- /dev/null +++ b/regression-test/suites/external_table_p0/paimon/test_paimon_deletion_vector_oss.groovy @@ -0,0 +1,60 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_paimon_deletion_vector_oss", "p0,external,doris,external_docker,external_docker_doris") { + + logger.info("start paimon test") + String enabled = context.config.otherConfigs.get("enablePaimonTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disabled paimon test") + return + } + + try { + String catalog_name = "test_paimon_deletion_vector_oss" + String aliYunAk = context.config.otherConfigs.get("aliYunAk") + String aliYunSk = context.config.otherConfigs.get("aliYunSk") + sql """drop catalog if exists ${catalog_name}""" + sql """create catalog if not exists ${catalog_name} properties ( + "type" = "paimon", + "paimon.catalog.type"="filesystem", + "warehouse" = "oss://doris-regression-bj/regression/paimon1", + "oss.access_key"="${aliYunAk}", + "oss.secret_key"="${aliYunSk}", + "oss.endpoint"="oss-cn-beijing.aliyuncs.com" + );""" + + sql """use `${catalog_name}`.`db1`""" + + def test_cases = { String force -> + sql """ set force_jni_scanner=${force} """ + qt_1 """select count(*) from deletion_vector_orc;""" + qt_2 """select count(*) from deletion_vector_parquet;""" + qt_3 """select count(*) from deletion_vector_orc where id > 2;""" + qt_4 """select count(*) from deletion_vector_parquet where id > 2;""" + qt_5 """select * from deletion_vector_orc where id > 2 order by id;""" + qt_6 """select * from deletion_vector_parquet where id > 2 order by id;""" + } + + test_cases("false") + test_cases("true") + + } finally { + sql """set force_jni_scanner=false""" + } + +} \ No newline at end of file