From ea5083bb74e3f62cd9a8cc6f39eb916987917557 Mon Sep 17 00:00:00 2001 From: Delena Malan Date: Wed, 19 Oct 2022 01:14:27 +0200 Subject: [PATCH 1/2] Fix backoff strategy docs --- .../error_handlers/composite_error_handler.py | 4 ++-- .../error_handlers/default_error_handler.py | 2 +- .../error-handling.md | 20 +++++++++---------- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/error_handlers/composite_error_handler.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/error_handlers/composite_error_handler.py index 0c2cfe5da878f..ffaed5bcb6065 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/error_handlers/composite_error_handler.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/error_handlers/composite_error_handler.py @@ -26,13 +26,13 @@ class CompositeErrorHandler(ErrorHandler, JsonSchemaMixin): - predicate: "{{ 'codase' in response }}" action: RETRY backoff_strategies: - - type: "ConstantBackoffStrategy" + - type: "ConstantBackoff" backoff_time_in_seconds: 5 - response_filters: - http_codes: [ 403 ] action: RETRY backoff_strategies: - - type: "ConstantBackoffStrategy" + - type: "ConstantBackoff" backoff_time_in_seconds: 10 Attributes: error_handlers (List[ErrorHandler]): list of error handlers diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/error_handlers/default_error_handler.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/error_handlers/default_error_handler.py index 4275d0ce6ba65..70168560cac2d 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/error_handlers/default_error_handler.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/error_handlers/default_error_handler.py @@ -41,7 +41,7 @@ class DefaultErrorHandler(ErrorHandler, JsonSchemaMixin): ` error_handler: backoff_strategies: - - type: "ConstantBackoffStrategy" + - type: "ConstantBackoff" backoff_time_in_seconds: 5 ` 3. retry on HTTP 404 diff --git a/docs/connector-development/config-based/understanding-the-yaml-file/error-handling.md b/docs/connector-development/config-based/understanding-the-yaml-file/error-handling.md index 2346cbd76db92..fdb3a6f07d61a 100644 --- a/docs/connector-development/config-based/understanding-the-yaml-file/error-handling.md +++ b/docs/connector-development/config-based/understanding-the-yaml-file/error-handling.md @@ -183,7 +183,7 @@ Schema: ### Constant Backoff -When using the `ConstantBackoffStrategy`, the requester will backoff with a constant interval. +When using the `ConstantBackoff` strategy, the requester will backoff with a constant interval. Schema: @@ -202,7 +202,7 @@ Schema: ### Wait time defined in header -When using the `WaitTimeFromHeaderBackoffStrategy`, the requester will backoff by an interval specified in the response header. +When using the `WaitTimeFromHeader`, the requester will backoff by an interval specified in the response header. In this example, the requester will backoff by the response's "wait_time" header value: Schema: @@ -230,7 +230,7 @@ requester: error_handler: <...> backoff_strategies: - - type: "WaitTimeFromHeaderBackoffStrategy" + - type: "WaitTimeFromHeader" header: "wait_time" ``` @@ -244,14 +244,14 @@ requester: error_handler: <...> backoff_strategies: - - type: "WaitTimeFromHeaderBackoffStrategy" + - type: "WaitTimeFromHeader" header: "wait_time" regex: "[-+]?\d+" ``` ### Wait until time defined in header -When using the `WaitUntilTimeFromHeaderBackoffStrategy`, the requester will backoff until the time specified in the response header. +When using the `WaitUntilTimeFromHeader` backoff strategy, the requester will backoff until the time specified in the response header. In this example, the requester will wait until the time specified in the "wait_until" header value: Schema: @@ -281,7 +281,7 @@ requester: error_handler: <...> backoff_strategies: - - type: "WaitUntilTimeFromHeaderBackoffStrategy" + - type: "WaitUntilTimeFromHeader" header: "wait_until" regex: "[-+]?\d+" min_wait: 5 @@ -302,9 +302,9 @@ requester: error_handler: <...> backoff_strategies: - - type: "WaitTimeFromHeaderBackoffStrategy" + - type: "WaitTimeFromHeader" header: "wait_time" - - type: "ConstantBackoffStrategy" + - type: "ConstantBackoff" backoff_time_in_seconds: 5 ``` @@ -340,13 +340,13 @@ requester: - predicate: "{{ 'code' in response }}" action: RETRY backoff_strategies: - - type: "ConstantBackoffStrategy" + - type: "ConstantBackoff" backoff_time_in_seconds: 5 - response_filters: - http_codes: [ 403 ] action: RETRY backoff_strategies: - - type: "ExponentialBackoffStrategy" + - type: "ExponentialBackoff" ``` ## More readings From 54be5225f96d6ac17dd1595bb514ede423936f78 Mon Sep 17 00:00:00 2001 From: Delena Malan Date: Wed, 19 Oct 2022 01:16:55 +0200 Subject: [PATCH 2/2] Update config_component_schema.json --- .../declarative/config_component_schema.json | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/config_component_schema.json b/airbyte-cdk/python/airbyte_cdk/sources/declarative/config_component_schema.json index f8ff83b6e52d9..5ae05dbf5fcf7 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/config_component_schema.json +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/config_component_schema.json @@ -648,7 +648,7 @@ } } ], - "description": "\n Error handler that sequentially iterates over a list of `ErrorHandler`s\n\n Sample config chaining 2 different retriers:\n error_handler:\n type: \"CompositeErrorHandler\"\n error_handlers:\n - response_filters:\n - predicate: \"{{ 'codase' in response }}\"\n action: RETRY\n backoff_strategies:\n - type: \"ConstantBackoffStrategy\"\n backoff_time_in_seconds: 5\n - response_filters:\n - http_codes: [ 403 ]\n action: RETRY\n backoff_strategies:\n - type: \"ConstantBackoffStrategy\"\n backoff_time_in_seconds: 10\n Attributes:\n error_handlers (List[ErrorHandler]): list of error handlers\n " + "description": "\n Error handler that sequentially iterates over a list of `ErrorHandler`s\n\n Sample config chaining 2 different retriers:\n error_handler:\n type: \"CompositeErrorHandler\"\n error_handlers:\n - response_filters:\n - predicate: \"{{ 'codase' in response }}\"\n action: RETRY\n backoff_strategies:\n - type: \"ConstantBackoff\"\n backoff_time_in_seconds: 5\n - response_filters:\n - http_codes: [ 403 ]\n action: RETRY\n backoff_strategies:\n - type: \"ConstantBackoff\"\n backoff_time_in_seconds: 10\n Attributes:\n error_handlers (List[ErrorHandler]): list of error handlers\n " }, "DefaultErrorHandler": { "allOf": [ @@ -677,16 +677,16 @@ "items": { "anyOf": [ { - "$ref": "#/definitions/ConstantBackoffStrategy" + "$ref": "#/definitions/ConstantBackoff" }, { - "$ref": "#/definitions/ExponentialBackoffStrategy" + "$ref": "#/definitions/ExponentialBackoff" }, { - "$ref": "#/definitions/WaitTimeFromHeaderBackoffStrategy" + "$ref": "#/definitions/WaitTimeFromHeader" }, { - "$ref": "#/definitions/WaitUntilTimeFromHeaderBackoffStrategy" + "$ref": "#/definitions/WaitUntilTimeFromHeader" } ] } @@ -694,7 +694,7 @@ } } ], - "description": "\n Default error handler.\n\n By default, the handler will only retry server errors (HTTP 5XX) and too many requests (HTTP 429) with exponential backoff.\n\n If the response is successful, then return SUCCESS\n Otherwise, iterate over the response_filters.\n If any of the filter match the response, then return the appropriate status.\n If the match is RETRY, then iterate sequentially over the backoff_strategies and return the first non-None backoff time.\n\n Sample configs:\n\n 1. retry 10 times\n `\n error_handler:\n max_retries: 10\n `\n 2. backoff for 5 seconds\n `\n error_handler:\n backoff_strategies:\n - type: \"ConstantBackoffStrategy\"\n backoff_time_in_seconds: 5\n `\n 3. retry on HTTP 404\n `\n error_handler:\n response_filters:\n - http_codes: [ 404 ]\n action: RETRY\n `\n 4. ignore HTTP 404\n `\n error_handler:\n - http_codes: [ 404 ]\n action: IGNORE\n `\n 5. retry if error message contains `retrythisrequest!` substring\n `\n error_handler:\n response_filters:\n - error_message_contain: \"retrythisrequest!\"\n action: IGNORE\n `\n 6. retry if 'code' is a field present in the response body\n `\n error_handler:\n response_filters:\n - predicate: \"{{ 'code' in response }}\"\n action: IGNORE\n `\n\n 7. ignore 429 and retry on 404\n `\n error_handler:\n - http_codes: [ 429 ]\n action: IGNORE\n - http_codes: [ 404 ]\n action: RETRY\n `\n\n Attributes:\n response_filters (Optional[List[HttpResponseFilter]]): response filters to iterate on\n max_retries (Optional[int]): maximum retry attempts\n backoff_strategies (Optional[List[BackoffStrategy]]): list of backoff strategies to use to determine how long\n to wait before retrying\n " + "description": "\n Default error handler.\n\n By default, the handler will only retry server errors (HTTP 5XX) and too many requests (HTTP 429) with exponential backoff.\n\n If the response is successful, then return SUCCESS\n Otherwise, iterate over the response_filters.\n If any of the filter match the response, then return the appropriate status.\n If the match is RETRY, then iterate sequentially over the backoff_strategies and return the first non-None backoff time.\n\n Sample configs:\n\n 1. retry 10 times\n `\n error_handler:\n max_retries: 10\n `\n 2. backoff for 5 seconds\n `\n error_handler:\n backoff_strategies:\n - type: \"ConstantBackoff\"\n backoff_time_in_seconds: 5\n `\n 3. retry on HTTP 404\n `\n error_handler:\n response_filters:\n - http_codes: [ 404 ]\n action: RETRY\n `\n 4. ignore HTTP 404\n `\n error_handler:\n - http_codes: [ 404 ]\n action: IGNORE\n `\n 5. retry if error message contains `retrythisrequest!` substring\n `\n error_handler:\n response_filters:\n - error_message_contain: \"retrythisrequest!\"\n action: IGNORE\n `\n 6. retry if 'code' is a field present in the response body\n `\n error_handler:\n response_filters:\n - predicate: \"{{ 'code' in response }}\"\n action: IGNORE\n `\n\n 7. ignore 429 and retry on 404\n `\n error_handler:\n - http_codes: [ 429 ]\n action: IGNORE\n - http_codes: [ 404 ]\n action: RETRY\n `\n\n Attributes:\n response_filters (Optional[List[HttpResponseFilter]]): response filters to iterate on\n max_retries (Optional[int]): maximum retry attempts\n backoff_strategies (Optional[List[BackoffStrategy]]): list of backoff strategies to use to determine how long\n to wait before retrying\n " }, "HttpResponseFilter": { "type": "object", @@ -745,7 +745,7 @@ }, "description": "InterpolatedBoolean(condition: str, options: dataclasses.InitVar[typing.Mapping[str, typing.Any]])" }, - "ConstantBackoffStrategy": { + "ConstantBackoff": { "allOf": [ { "$ref": "#/definitions/BackoffStrategy" @@ -767,7 +767,7 @@ "properties": {}, "description": "\n Backoff strategy defining how long to wait before retrying a request that resulted in an error.\n " }, - "ExponentialBackoffStrategy": { + "ExponentialBackoff": { "allOf": [ { "$ref": "#/definitions/BackoffStrategy" @@ -784,7 +784,7 @@ ], "description": "\n Backoff strategy with an exponential backoff interval\n\n Attributes:\n factor (float): multiplicative factor\n " }, - "WaitTimeFromHeaderBackoffStrategy": { + "WaitTimeFromHeader": { "allOf": [ { "$ref": "#/definitions/BackoffStrategy" @@ -804,7 +804,7 @@ ], "description": "\n Extract wait time from http header\n\n Attributes:\n header (str): header to read wait time from\n regex (Optional[str]): optional regex to apply on the header to extract its value\n " }, - "WaitUntilTimeFromHeaderBackoffStrategy": { + "WaitUntilTimeFromHeader": { "allOf": [ { "$ref": "#/definitions/BackoffStrategy"