diff --git a/providers/edge3/provider.yaml b/providers/edge3/provider.yaml index a4698c04ef7f4..cb2cda5e53b5f 100644 --- a/providers/edge3/provider.yaml +++ b/providers/edge3/provider.yaml @@ -132,6 +132,15 @@ config: type: integer example: ~ default: "524288" + push_logs: + description: | + Flag to enable or disable pushing of log files from edge worker to the central site. + When enabled, edge workers will upload task log files in chunks to the central Airflow site. + When disabled, logs will only be available locally on the edge worker. + version_added: 1.5.0 + type: boolean + example: "True" + default: "True" worker_umask: description: | The default umask to use for edge worker when run in daemon mode diff --git a/providers/edge3/src/airflow/providers/edge3/cli/worker.py b/providers/edge3/src/airflow/providers/edge3/cli/worker.py index d1e0bcdcf2896..7090f627ef0c4 100644 --- a/providers/edge3/src/airflow/providers/edge3/cli/worker.py +++ b/providers/edge3/src/airflow/providers/edge3/cli/worker.py @@ -348,7 +348,11 @@ def check_running_jobs(self) -> None: else: used_concurrency += job.edge_job.concurrency_slots - if job.logfile.exists() and job.logfile.stat().st_size > job.logsize: + if ( + conf.getboolean("edge", "push_logs") + and job.logfile.exists() + and job.logfile.stat().st_size > job.logsize + ): with job.logfile.open("rb") as logfile: push_log_chunk_size = conf.getint("edge", "push_log_chunk_size") logfile.seek(job.logsize, os.SEEK_SET) diff --git a/providers/edge3/src/airflow/providers/edge3/get_provider_info.py b/providers/edge3/src/airflow/providers/edge3/get_provider_info.py index 5892ebc76ac23..b8a3fb890d0b1 100644 --- a/providers/edge3/src/airflow/providers/edge3/get_provider_info.py +++ b/providers/edge3/src/airflow/providers/edge3/get_provider_info.py @@ -93,6 +93,13 @@ def get_provider_info(): "example": None, "default": "524288", }, + "push_logs": { + "description": "Flag to enable or disable pushing of log files from edge worker to the central site.\nWhen enabled, edge workers will upload task log files in chunks to the central Airflow site.\nWhen disabled, logs will only be available locally on the edge worker.\n", + "version_added": "1.5.0", + "type": "boolean", + "example": "True", + "default": "True", + }, "worker_umask": { "description": "The default umask to use for edge worker when run in daemon mode\n\nThis controls the file-creation mode mask which determines the initial value of file permission bits\nfor newly created files.\n\nThis value is treated as an octal-integer.\n", "version_added": None,