From 14ba3a4cd7fd17c4019fda48cf3c79d9e6b56225 Mon Sep 17 00:00:00 2001 From: Socrates Date: Fri, 28 Mar 2025 20:05:57 +0800 Subject: [PATCH 1/3] covert paimon uri in fe --- .../doris/datasource/paimon/source/PaimonScanNode.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java index d1e4d1601c412e..a8d9c0c42e5173 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java @@ -210,7 +210,11 @@ private void setPaimonParams(TFileRangeDesc rangeDesc, PaimonSplit paimonSplit) if (optDeletionFile.isPresent()) { DeletionFile deletionFile = optDeletionFile.get(); TPaimonDeletionFileDesc tDeletionFile = new TPaimonDeletionFileDesc(); - tDeletionFile.setPath(deletionFile.path()); + // convert the deletion file uri to make sure FileReader can read it in be + LocationPath locationPath = new LocationPath(deletionFile.path(), + source.getCatalog().getProperties()); + String path = locationPath.toStorageLocation().toString(); + tDeletionFile.setPath(path); tDeletionFile.setOffset(deletionFile.offset()); tDeletionFile.setLength(deletionFile.length()); fileDesc.setDeletionFile(tDeletionFile); From e2dfb37e42de971240a7831c083ee0b921547630 Mon Sep 17 00:00:00 2001 From: Socrates Date: Tue, 1 Apr 2025 01:21:03 +0800 Subject: [PATCH 2/3] add case --- .../test_paimon_deletion_vector_oss.out | 73 +++++++++++++ .../test_paimon_deletion_vector_oss.groovy | 100 ++++++++++++++++++ 2 files changed, 173 insertions(+) create mode 100644 regression-test/data/external_table_p0/paimon/test_paimon_deletion_vector_oss.out create mode 100644 regression-test/suites/external_table_p0/paimon/test_paimon_deletion_vector_oss.groovy diff --git a/regression-test/data/external_table_p0/paimon/test_paimon_deletion_vector_oss.out b/regression-test/data/external_table_p0/paimon/test_paimon_deletion_vector_oss.out new file mode 100644 index 00000000000000..f0b1e92a088538 --- /dev/null +++ b/regression-test/data/external_table_p0/paimon/test_paimon_deletion_vector_oss.out @@ -0,0 +1,73 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !1 -- +3 + +-- !2 -- +3 + +-- !3 -- +2 + +-- !4 -- +2 + +-- !5 -- +3 3_1 +4 4_1 + +-- !6 -- +3 3_1 +4 4_1 + +-- !7 -- +1 Paimon 5 +4 Venti 25 +5 Kaeya 28 +6 Jean 26 +7 Diluc 30 +8 Razor 18 +9 Mona 19 +10 Barbara 16 + +-- !8 -- +8 + +-- !9 -- +7 + +-- !1 -- +3 + +-- !2 -- +3 + +-- !3 -- +2 + +-- !4 -- +2 + +-- !5 -- +3 3_1 +4 4_1 + +-- !6 -- +3 3_1 +4 4_1 + +-- !7 -- +1 Paimon 5 +4 Venti 25 +5 Kaeya 28 +6 Jean 26 +7 Diluc 30 +8 Razor 18 +9 Mona 19 +10 Barbara 16 + +-- !8 -- +8 + +-- !9 -- +7 + diff --git a/regression-test/suites/external_table_p0/paimon/test_paimon_deletion_vector_oss.groovy b/regression-test/suites/external_table_p0/paimon/test_paimon_deletion_vector_oss.groovy new file mode 100644 index 00000000000000..ce26cba8146c7b --- /dev/null +++ b/regression-test/suites/external_table_p0/paimon/test_paimon_deletion_vector_oss.groovy @@ -0,0 +1,100 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_paimon_deletion_vector_oss", "p0,external,doris,external_docker,external_docker_doris") { + + logger.info("start paimon test") + String enabled = context.config.otherConfigs.get("enablePaimonTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disabled paimon test") + return + } + + try { + String catalog_name = "test_paimon_deletion_vector_oss" + String aliYunAk = context.config.otherConfigs.get("aliYunAk") + String aliYunSk = context.config.otherConfigs.get("aliYunSk") + sql """drop catalog if exists ${catalog_name}""" + sql """create catalog if not exists ${catalog_name} properties ( + "type" = "paimon", + "paimon.catalog.type"="filesystem", + "warehouse" = "oss://doris-regression-bj/regression/paimon1", + "oss.access_key"="${aliYunAk}", + "oss.secret_key"="${aliYunSk}", + "oss.endpoint"="oss-cn-beijing.aliyuncs.com" + );""" + + sql """use `${catalog_name}`.`db1`""" + + def test_cases = { String force -> + sql """ set force_jni_scanner=${force} """ + qt_1 """select count(*) from deletion_vector_orc;""" + qt_2 """select count(*) from deletion_vector_parquet;""" + qt_3 """select count(*) from deletion_vector_orc where id > 2;""" + qt_4 """select count(*) from deletion_vector_parquet where id > 2;""" + qt_5 """select * from deletion_vector_orc where id > 2 order by id;""" + qt_6 """select * from deletion_vector_parquet where id > 2 order by id;""" + qt_7 """select * from deletion_vector_table_1_0 order by id;""" + qt_8 """select count(*) from deletion_vector_table_1_0;""" + qt_9 """select count(*) from deletion_vector_table_1_0 where id > 2;""" + } + + def test_table_count_push_down = { String force -> + sql """ set force_jni_scanner=${force} """ + explain { + sql("select count(*) from deletion_vector_orc;") + contains "pushdown agg=COUNT (-1)" + } + explain { + sql("select count(*) from deletion_vector_parquet;") + contains "pushdown agg=COUNT (-1)" + } + explain { + sql("select count(*) from deletion_vector_table_1_0;") + contains "pushdown agg=COUNT (8)" + } + } + + def test_not_table_count_push_down = { String force -> + sql """ set enable_count_push_down_for_external_table=false; """ + sql """ set force_jni_scanner=${force} """ + explain { + sql("select count(*) from deletion_vector_orc;") + contains "pushdown agg=NONE" + } + explain { + sql("select count(*) from deletion_vector_parquet;") + contains "pushdown agg=NONE" + } + explain { + sql("select count(*) from deletion_vector_table_1_0;") + contains "pushdown agg=NONE" + } + } + + test_cases("false") + test_cases("true") + test_table_count_push_down("false") + test_table_count_push_down("true") + test_not_table_count_push_down("false") + test_not_table_count_push_down("true") + } finally { + sql """ set enable_count_push_down_for_external_table=true; """ + sql """set force_jni_scanner=false""" + } + +} \ No newline at end of file From dbc9fabaedad2be9b4e52e244be776abd2593de3 Mon Sep 17 00:00:00 2001 From: Socrates Date: Tue, 1 Apr 2025 10:33:49 +0800 Subject: [PATCH 3/3] fix case --- .../test_paimon_deletion_vector_oss.groovy | 42 +------------------ 1 file changed, 1 insertion(+), 41 deletions(-) diff --git a/regression-test/suites/external_table_p0/paimon/test_paimon_deletion_vector_oss.groovy b/regression-test/suites/external_table_p0/paimon/test_paimon_deletion_vector_oss.groovy index ce26cba8146c7b..55fc903a31b3a3 100644 --- a/regression-test/suites/external_table_p0/paimon/test_paimon_deletion_vector_oss.groovy +++ b/regression-test/suites/external_table_p0/paimon/test_paimon_deletion_vector_oss.groovy @@ -48,52 +48,12 @@ suite("test_paimon_deletion_vector_oss", "p0,external,doris,external_docker,exte qt_4 """select count(*) from deletion_vector_parquet where id > 2;""" qt_5 """select * from deletion_vector_orc where id > 2 order by id;""" qt_6 """select * from deletion_vector_parquet where id > 2 order by id;""" - qt_7 """select * from deletion_vector_table_1_0 order by id;""" - qt_8 """select count(*) from deletion_vector_table_1_0;""" - qt_9 """select count(*) from deletion_vector_table_1_0 where id > 2;""" - } - - def test_table_count_push_down = { String force -> - sql """ set force_jni_scanner=${force} """ - explain { - sql("select count(*) from deletion_vector_orc;") - contains "pushdown agg=COUNT (-1)" - } - explain { - sql("select count(*) from deletion_vector_parquet;") - contains "pushdown agg=COUNT (-1)" - } - explain { - sql("select count(*) from deletion_vector_table_1_0;") - contains "pushdown agg=COUNT (8)" - } - } - - def test_not_table_count_push_down = { String force -> - sql """ set enable_count_push_down_for_external_table=false; """ - sql """ set force_jni_scanner=${force} """ - explain { - sql("select count(*) from deletion_vector_orc;") - contains "pushdown agg=NONE" - } - explain { - sql("select count(*) from deletion_vector_parquet;") - contains "pushdown agg=NONE" - } - explain { - sql("select count(*) from deletion_vector_table_1_0;") - contains "pushdown agg=NONE" - } } test_cases("false") test_cases("true") - test_table_count_push_down("false") - test_table_count_push_down("true") - test_not_table_count_push_down("false") - test_not_table_count_push_down("true") + } finally { - sql """ set enable_count_push_down_for_external_table=true; """ sql """set force_jni_scanner=false""" }