From a3db44f6b69f84af594a9263984649295d6044d5 Mon Sep 17 00:00:00 2001
From: Socrates <suxiaogang223@icloud.com>
Date: Fri, 11 Jul 2025 06:36:46 +0800
Subject: [PATCH] branch-2.1: [fix](tvf) support compressed json file for tvf
 and refactor code (#51983) (#53063)

bp: #51983
---
 be/src/vec/exec/format/json/new_json_reader.cpp   |   4 +++-
 .../json_format_test/simple_object_json.json.gz   | Bin 0 -> 211 bytes
 .../data/external_table_p0/tvf/test_hdfs_tvf.out  |  14 ++++++++++++++
 .../external_table_p0/tvf/test_hdfs_tvf.groovy    |  10 ++++++++++
 4 files changed, 27 insertions(+), 1 deletion(-)
 create mode 100644 docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/json_format_test/simple_object_json.json.gz

diff --git a/be/src/vec/exec/format/json/new_json_reader.cpp b/be/src/vec/exec/format/json/new_json_reader.cpp
index c8969c6d4c314e..fb1b04e867c44b 100644
--- a/be/src/vec/exec/format/json/new_json_reader.cpp
+++ b/be/src/vec/exec/format/json/new_json_reader.cpp
@@ -258,7 +258,9 @@ Status NewJsonReader::get_columns(std::unordered_map<std::string, TypeDescriptor
 Status NewJsonReader::get_parsed_schema(std::vector<std::string>* col_names,
                                         std::vector<TypeDescriptor>* col_types) {
     RETURN_IF_ERROR(_get_range_params());
-
+    // create decompressor.
+    // _decompressor may be nullptr if this is not a compressed file
+    RETURN_IF_ERROR(Decompressor::create_decompressor(_file_compress_type, &_decompressor));
     RETURN_IF_ERROR(_open_file_reader(true));
     if (_read_json_by_line) {
         RETURN_IF_ERROR(_open_line_reader());
diff --git a/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/json_format_test/simple_object_json.json.gz b/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/json_format_test/simple_object_json.json.gz
new file mode 100644
index 0000000000000000000000000000000000000000..8a6db90241ffc25e12d351e459c1f4ce374289af
GIT binary patch
literal 211
zcmV;^04)C>iwFo!<PB&519NF@aBO8?Z(?d?V{~6?b8l`gYIARH0DaNH4uc>J2H?9-
zVL0y~h^TuTrVLmN9ya2p@!f}6QoGdcKth_2KP_FD@7X!$r<`^E@k&>AG5+R*Wm{?N
zO)7~}R;uP}wtTce`tEeFPJ6-y@Dgzw6QA7}i7&y+uXq#YbO@KgE5zNe^a{8_Jm_#Z
zssddjJ;fAHYz?+Swo5(Rp7zmzZjpZagWwh@FYt)wYUpqCV%KPyAr1qf`y#pc6TH1$
Nvp1P-u<aoN004zzW0(K{

literal 0
HcmV?d00001

diff --git a/regression-test/data/external_table_p0/tvf/test_hdfs_tvf.out b/regression-test/data/external_table_p0/tvf/test_hdfs_tvf.out
index a8f5dcf5396932..04ec58cdbaefc6 100644
--- a/regression-test/data/external_table_p0/tvf/test_hdfs_tvf.out
+++ b/regression-test/data/external_table_p0/tvf/test_hdfs_tvf.out
@@ -286,6 +286,20 @@
 8	chengdu	2345678
 9	xian	2345679
 
+-- !json_compressed --
+1	beijing	2345671
+10	hefei	23456710
+11	\N	23456711
+12	hefei	\N
+2	shanghai	2345672
+3	guangzhou	2345673
+4	shenzhen	2345674
+5	hangzhou	2345675
+6	nanjing	2345676
+7	wuhan	2345677
+8	chengdu	2345678
+9	xian	2345679
+
 -- !json_limit1 --
 1	beijing	2345671
 10	hefei	23456710
diff --git a/regression-test/suites/external_table_p0/tvf/test_hdfs_tvf.groovy b/regression-test/suites/external_table_p0/tvf/test_hdfs_tvf.groovy
index 74cb1e320aaa16..8bc8194843d9c6 100644
--- a/regression-test/suites/external_table_p0/tvf/test_hdfs_tvf.groovy
+++ b/regression-test/suites/external_table_p0/tvf/test_hdfs_tvf.groovy
@@ -143,6 +143,16 @@ suite("test_hdfs_tvf","external,hive,tvf,external_docker") {
                         "strip_outer_array" = "false",
                         "read_json_by_line" = "true") order by id; """
 
+            uri = "${defaultFS}" + "/user/doris/preinstalled_data/json_format_test/simple_object_json.json.gz"
+            format = "json"
+            qt_json_compressed """ select * from HDFS(
+                        "uri" = "${uri}",
+                        "hadoop.username" = "${hdfsUserName}",
+                        "format" = "${format}",
+                        "compress_type" = "GZ",
+                        "strip_outer_array" = "false",
+                        "read_json_by_line" = "true") order by id; """
+
 
            uri = "${defaultFS}" + "/user/doris/preinstalled_data/json_format_test/simple_object_json.json"
             format = "json"