From 43b1d63c168c877d384f8ea529575ad1f310c18b Mon Sep 17 00:00:00 2001 From: Kacper Muda Date: Thu, 6 Mar 2025 13:36:35 +0100 Subject: [PATCH 1/2] chore: Update description of `execution_timeout` in OpenLineage provider.yaml --- providers/openlineage/provider.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/providers/openlineage/provider.yaml b/providers/openlineage/provider.yaml index f10e80b86009e..23899a7a438be 100644 --- a/providers/openlineage/provider.yaml +++ b/providers/openlineage/provider.yaml @@ -159,6 +159,9 @@ config: execution_timeout: description: | Maximum amount of time (in seconds) that OpenLineage can spend executing metadata extraction. + Note that other configurations, sometimes with higher priority, may also affect how much time + OpenLineage has for execution, such as core.task_success_overtime. For more details, see: + https://airflow.apache.org/docs/apache-airflow/stable/configurations-ref.html#task-success-overtime default: "10" example: ~ type: integer From 7bdbc371371d0f98df3d0f5a94f527fe41d4c7c2 Mon Sep 17 00:00:00 2001 From: Kacper Muda Date: Fri, 7 Mar 2025 08:51:02 +0100 Subject: [PATCH 2/2] chore: Update description of execution_timeout in OpenLineage provider.yaml --- providers/openlineage/provider.yaml | 20 +++++++++++-------- .../openlineage/get_provider_info.py | 6 +++--- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/providers/openlineage/provider.yaml b/providers/openlineage/provider.yaml index 23899a7a438be..675045adc4573 100644 --- a/providers/openlineage/provider.yaml +++ b/providers/openlineage/provider.yaml @@ -66,8 +66,9 @@ config: openlineage: description: | This section applies settings for OpenLineage integration. - More about configuration and it's precedence can be found at - https://airflow.apache.org/docs/apache-airflow-providers-openlineage/stable/guides/user.html#transport-setup + More about configuration and it's precedence can be found in the `user's guide + `_. + options: disabled: description: | @@ -125,9 +126,9 @@ config: default: "" transport: description: | - Pass OpenLineage Client transport configuration as JSON string. It should contain type of the - transport and additional options (different for each transport type). For more details see: - https://openlineage.io/docs/client/python/#built-in-transport-types + Pass OpenLineage Client transport configuration as a JSON string, including the transport type + and any additional options specific to that type, as described in `OpenLineage docs + `_. Currently supported types are: @@ -135,6 +136,8 @@ config: * Kafka * Console * File + * Composite + * Custom type: string example: '{"type": "http", "url": "http://localhost:5000", "endpoint": "api/v1/lineage"}' default: "" @@ -159,9 +162,10 @@ config: execution_timeout: description: | Maximum amount of time (in seconds) that OpenLineage can spend executing metadata extraction. - Note that other configurations, sometimes with higher priority, may also affect how much time - OpenLineage has for execution, such as core.task_success_overtime. For more details, see: - https://airflow.apache.org/docs/apache-airflow/stable/configurations-ref.html#task-success-overtime + Note that other configurations, sometimes with higher priority, such as + `[core] task_success_overtime + `_, + may also affect how much time OpenLineage has for execution. default: "10" example: ~ type: integer diff --git a/providers/openlineage/src/airflow/providers/openlineage/get_provider_info.py b/providers/openlineage/src/airflow/providers/openlineage/get_provider_info.py index bfca6e1e544be..4d9fa7a200674 100644 --- a/providers/openlineage/src/airflow/providers/openlineage/get_provider_info.py +++ b/providers/openlineage/src/airflow/providers/openlineage/get_provider_info.py @@ -72,7 +72,7 @@ def get_provider_info(): ], "config": { "openlineage": { - "description": "This section applies settings for OpenLineage integration.\nMore about configuration and it's precedence can be found at\nhttps://airflow.apache.org/docs/apache-airflow-providers-openlineage/stable/guides/user.html#transport-setup\n", + "description": "This section applies settings for OpenLineage integration.\nMore about configuration and it's precedence can be found in the `user's guide\n`_.\n", "options": { "disabled": { "description": "Disable sending events without uninstalling the OpenLineage Provider by setting this to true.\n", @@ -124,7 +124,7 @@ def get_provider_info(): "default": "", }, "transport": { - "description": "Pass OpenLineage Client transport configuration as JSON string. It should contain type of the\ntransport and additional options (different for each transport type). For more details see:\nhttps://openlineage.io/docs/client/python/#built-in-transport-types\n\nCurrently supported types are:\n\n * HTTP\n * Kafka\n * Console\n * File\n", + "description": "Pass OpenLineage Client transport configuration as a JSON string, including the transport type\nand any additional options specific to that type, as described in `OpenLineage docs\n`_.\n\nCurrently supported types are:\n\n * HTTP\n * Kafka\n * Console\n * File\n * Composite\n * Custom\n", "type": "string", "example": '{"type": "http", "url": "http://localhost:5000", "endpoint": "api/v1/lineage"}', "default": "", @@ -145,7 +145,7 @@ def get_provider_info(): "version_added": "1.8.0", }, "execution_timeout": { - "description": "Maximum amount of time (in seconds) that OpenLineage can spend executing metadata extraction.\n", + "description": "Maximum amount of time (in seconds) that OpenLineage can spend executing metadata extraction.\nNote that other configurations, sometimes with higher priority, such as\n`[core] task_success_overtime\n`_,\nmay also affect how much time OpenLineage has for execution.\n", "default": "10", "example": None, "type": "integer",