From 17f615883c192aae1fce75fe8a6038c02f395f03 Mon Sep 17 00:00:00 2001 From: roach231428 Date: Tue, 20 May 2025 11:27:43 +0800 Subject: [PATCH 1/2] Fix: use relative HDFS path instead of full URL for remote_base --- .../src/airflow/config_templates/airflow_local_settings.py | 2 +- .../src/airflow/providers/apache/hdfs/log/hdfs_task_handler.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/airflow-core/src/airflow/config_templates/airflow_local_settings.py b/airflow-core/src/airflow/config_templates/airflow_local_settings.py index 3cade5678261c..b93f786e1eadc 100644 --- a/airflow-core/src/airflow/config_templates/airflow_local_settings.py +++ b/airflow-core/src/airflow/config_templates/airflow_local_settings.py @@ -236,7 +236,7 @@ **( { "base_log_folder": BASE_LOG_FOLDER, - "remote_base": remote_base_log_folder, + "remote_base": urlsplit(remote_base_log_folder).path, "delete_local_copy": delete_local_copy, } | remote_task_handler_kwargs diff --git a/providers/apache/hdfs/src/airflow/providers/apache/hdfs/log/hdfs_task_handler.py b/providers/apache/hdfs/src/airflow/providers/apache/hdfs/log/hdfs_task_handler.py index 4de610203416c..b0b89aad6c70c 100644 --- a/providers/apache/hdfs/src/airflow/providers/apache/hdfs/log/hdfs_task_handler.py +++ b/providers/apache/hdfs/src/airflow/providers/apache/hdfs/log/hdfs_task_handler.py @@ -94,7 +94,7 @@ def __init__(self, base_log_folder: str, hdfs_log_folder: str, **kwargs): self.upload_on_close = True self.io = HdfsRemoteLogIO( - remote_base=hdfs_log_folder, + remote_base=self.remote_base, base_log_folder=base_log_folder, delete_local_copy=kwargs.get( "delete_local_copy", conf.getboolean("logging", "delete_local_logs") From 4067f6539972ad7c2a26fec93fe0f2ec7c159bea Mon Sep 17 00:00:00 2001 From: roach231428 Date: Fri, 30 May 2025 09:21:48 +0800 Subject: [PATCH 2/2] fix: correct mistakenly edited block in remote base log provider --- .../src/airflow/config_templates/airflow_local_settings.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/airflow-core/src/airflow/config_templates/airflow_local_settings.py b/airflow-core/src/airflow/config_templates/airflow_local_settings.py index b93f786e1eadc..cb1cc364e2ca6 100644 --- a/airflow-core/src/airflow/config_templates/airflow_local_settings.py +++ b/airflow-core/src/airflow/config_templates/airflow_local_settings.py @@ -236,7 +236,7 @@ **( { "base_log_folder": BASE_LOG_FOLDER, - "remote_base": urlsplit(remote_base_log_folder).path, + "remote_base": remote_base_log_folder, "delete_local_copy": delete_local_copy, } | remote_task_handler_kwargs @@ -250,7 +250,7 @@ **( { "base_log_folder": BASE_LOG_FOLDER, - "remote_base": remote_base_log_folder, + "remote_base": urlsplit(remote_base_log_folder).path, "delete_local_copy": delete_local_copy, } | remote_task_handler_kwargs