-
Notifications
You must be signed in to change notification settings - Fork 4.2k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
🐛 Source Harvest: Improve HTTP Availability (#35541)
- Loading branch information
Showing
10 changed files
with
158 additions
and
85 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
26 changes: 26 additions & 0 deletions
26
airbyte-integrations/connectors/source-harvest/source_harvest/availability_strategy.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,26 @@ | ||
| # Copyright (c) 2023 Airbyte, Inc., all rights reserved. | ||
|
|
||
| import logging | ||
| from typing import Dict | ||
|
|
||
| import requests | ||
| from airbyte_cdk.sources import Source | ||
| from airbyte_cdk.sources.streams import Stream | ||
| from airbyte_cdk.sources.streams.http.availability_strategy import HttpAvailabilityStrategy | ||
| from requests import HTTPError | ||
|
|
||
|
|
||
| class HarvestAvailabilityStrategy(HttpAvailabilityStrategy): | ||
| """ | ||
| This class is tested as part of test_source.check_connection | ||
| """ | ||
|
|
||
| def reasons_for_unavailable_status_codes( | ||
| self, stream: Stream, logger: logging.Logger, source: Source, error: HTTPError | ||
| ) -> Dict[int, str]: | ||
| reasons_for_codes: Dict[int, str] = { | ||
| requests.codes.UNAUTHORIZED: "Please ensure your credentials are valid.", | ||
| requests.codes.FORBIDDEN: "This is most likely due to insufficient permissions on the credentials in use.", | ||
| requests.codes.NOT_FOUND: "Please ensure that your account ID is properly set. If it is the case and you are still seeing this error, please contact Airbyte support.", | ||
| } | ||
| return reasons_for_codes |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
10 changes: 4 additions & 6 deletions
10
...te-integrations/connectors/source-harvest/unit_tests/integration/test_invoice_messages.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
8 changes: 3 additions & 5 deletions
8
airbyte-integrations/connectors/source-harvest/unit_tests/integration/test_invoices.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
75 changes: 75 additions & 0 deletions
75
airbyte-integrations/connectors/source-harvest/unit_tests/test_source.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,75 @@ | ||
| # Copyright (c) 2023 Airbyte, Inc., all rights reserved. | ||
|
|
||
| import unittest | ||
| from unittest.mock import Mock, patch | ||
|
|
||
| import pytest | ||
| import requests | ||
| from airbyte_cdk import AirbyteLogger | ||
| from airbyte_cdk.models import FailureType | ||
| from airbyte_cdk.utils import AirbyteTracedException | ||
| from config import ConfigBuilder | ||
| from requests import HTTPError | ||
| from source_harvest.source import SourceHarvest | ||
|
|
||
|
|
||
| def _a_response(status_code: int) -> requests.Response: | ||
| response = Mock(spec=requests.Response) | ||
| response.status_code = status_code | ||
| response.url = "any url" | ||
| response.reason = "any reason" | ||
| return response | ||
|
|
||
|
|
||
| class SourceTest(unittest.TestCase): | ||
|
|
||
| def setUp(self) -> None: | ||
| self._source = SourceHarvest() | ||
| self._logger = Mock(spec=AirbyteLogger) | ||
| self._config = ConfigBuilder().build() | ||
|
|
||
| def test_given_config_with_client_id_without_account_id_when_check_connection_then_raise_config_error(self) -> None: | ||
| config = ConfigBuilder().with_client_id("a client id").build() | ||
| config.pop("account_id") | ||
|
|
||
| with pytest.raises(AirbyteTracedException) as exception_trace: | ||
| self._source.check_connection(self._logger, config) | ||
| assert exception_trace.value.failure_type == FailureType.config_error | ||
|
|
||
| def test_given_config_no_authentication_in_config_when_check_connection_then_raise_config_error(self) -> None: | ||
| config = ConfigBuilder().build() | ||
| config["credentials"].pop("api_token", None) | ||
| config["credentials"].pop("client_id", None) | ||
|
|
||
| with pytest.raises(AirbyteTracedException) as exception_trace: | ||
| self._source.check_connection(self._logger, config) | ||
| assert exception_trace.value.failure_type == FailureType.config_error | ||
|
|
||
| @patch("source_harvest.source.Users.read_records") | ||
| def test_given_400_http_error_when_check_connection_then_raise_non_config_error(self, mocked_user_read_records) -> None: | ||
| """ | ||
| Following https://github.com/airbytehq/airbyte/pull/35305 where no page alerts were emitted | ||
| """ | ||
| mocked_user_read_records.side_effect = HTTPError(response=_a_response(400)) | ||
|
|
||
| with pytest.raises(Exception) as exception: | ||
| self._source.check_connection(self._logger, self._config) | ||
| assert not isinstance(exception, AirbyteTracedException) or exception.failure_type != FailureType.config_error | ||
|
|
||
| @patch("source_harvest.source.Users.read_records") | ||
| def test_given_401_http_error_when_check_connection_then_is_not_available(self, mocked_user_read_records) -> None: | ||
| mocked_user_read_records.side_effect = HTTPError(response=_a_response(401)) | ||
| is_available, _ = self._source.check_connection(self._logger, self._config) | ||
| assert not is_available | ||
|
|
||
| @patch("source_harvest.source.Users.read_records") | ||
| def test_given_403_http_error_when_check_connection_then_is_not_available(self, mocked_user_read_records) -> None: | ||
| mocked_user_read_records.side_effect = HTTPError(response=_a_response(403)) | ||
| is_available, _ = self._source.check_connection(self._logger, self._config) | ||
| assert not is_available | ||
|
|
||
| @patch("source_harvest.source.Users.read_records") | ||
| def test_given_404_http_error_when_check_connection_then_is_not_available(self, mocked_user_read_records) -> None: | ||
| mocked_user_read_records.side_effect = HTTPError(response=_a_response(404)) | ||
| is_available, _ = self._source.check_connection(self._logger, self._config) | ||
| assert not is_available |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters