From d62a29be73a342ccfc11d498cc2816fab446a442 Mon Sep 17 00:00:00 2001 From: Shishir Verma Date: Wed, 26 Apr 2023 01:03:43 +0530 Subject: [PATCH 1/4] use api url corresponding to region specified in the config --- .../source-genesys/source_genesys/source.py | 25 ++++++++++++++++--- .../source-genesys/unit_tests/test_source.py | 5 ++++ .../source-genesys/unit_tests/test_streams.py | 16 ++++++++---- 3 files changed, 38 insertions(+), 8 deletions(-) diff --git a/airbyte-integrations/connectors/source-genesys/source_genesys/source.py b/airbyte-integrations/connectors/source-genesys/source_genesys/source.py index 73ab86a9059d..e4e4dba98432 100644 --- a/airbyte-integrations/connectors/source-genesys/source_genesys/source.py +++ b/airbyte-integrations/connectors/source-genesys/source_genesys/source.py @@ -17,7 +17,8 @@ class GenesysStream(HttpStream, ABC): url_base = "https://api.mypurecloud.com.au/api/v2/" page_size = 500 - def __init__(self, *args, **kwargs): + def __init__(self, api_base_url, *args, **kwargs): + self.url_base = api_base_url + "/api/v2/" super().__init__(*args, **kwargs) def backoff_time(self, response: requests.Response) -> Optional[int]: @@ -29,7 +30,8 @@ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, response_json = response.json() if response_json.get("nextUri"): - next_query_string = urllib.parse.urlsplit(response_json.get("nextUri")).query + next_query_string = urllib.parse.urlsplit( + response_json.get("nextUri")).query return dict(urllib.parse.parse_qsl(next_query_string)) def request_params( @@ -268,7 +270,24 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: "Asia Pacific (Sydney)": "https://login.mypurecloud.com.au", } base_url = GENESYS_TENANT_ENDPOINT_MAP.get(config["tenant_endpoint"]) - args = {"authenticator": GenesysOAuthAuthenticator(base_url, config["client_id"], config["client_secret"])} + + GENESYS_API_SERVER_MAP: Dict = { + "Americas (US East)": "https://api.mypurecloud.com", + "Americas (US East 2)": "https://api.use2.us-gov-pure.cloud", + "Americas (US West)": "https://api.usw2.pure.cloud", + "Americas (Canada)": "https://api.cac1.pure.cloud", + "Americas (São Paulo)": "https://api.sae1.pure.cloud", + "EMEA (Frankfurt)": "https://api.mypurecloud.de", + "EMEA (Dublin)": "https://api.mypurecloud.ie", + "EMEA (London)": "https://api.euw2.pure.cloud", + "Asia Pacific (Mumbai)": "https://api.aps1.pure.cloud", + "Asia Pacific (Seoul)": "https://api.apne2.pure.cloud", + "Asia Pacific (Sydney)": "https://api.mypurecloud.com.au", + } + + api_base_url = GENESYS_API_SERVER_MAP.get(config["tenant_endpoint"]) + args = {"api_base_url": api_base_url, "authenticator": GenesysOAuthAuthenticator( + base_url, config["client_id"], config["client_secret"])} # response = self.get_connection_response(config) # response.raise_for_status() diff --git a/airbyte-integrations/connectors/source-genesys/unit_tests/test_source.py b/airbyte-integrations/connectors/source-genesys/unit_tests/test_source.py index 84e982fe4802..d46cbd327756 100644 --- a/airbyte-integrations/connectors/source-genesys/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-genesys/unit_tests/test_source.py @@ -17,7 +17,12 @@ def test_check_connection(mocker): def test_streams(mocker): source = SourceGenesys() config_mock = MagicMock() + config_mock.__getitem__.side_effect = lambda key: "Americas (US East)" if key == "tenant_endpoint" else None SourceGenesys.get_connection_response = MagicMock() streams = source.streams(config_mock) expected_streams_number = 16 assert len(streams) == expected_streams_number + + expected_url_base = "https://api.mypurecloud.com/api/v2/" + for stream in streams: + assert stream.url_base == expected_url_base diff --git a/airbyte-integrations/connectors/source-genesys/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-genesys/unit_tests/test_streams.py index 3c16b087c95d..6bdc49743fc1 100644 --- a/airbyte-integrations/connectors/source-genesys/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-genesys/unit_tests/test_streams.py @@ -18,20 +18,20 @@ def patch_base_class(mocker): def test_request_params(patch_base_class): - stream = GenesysStream() + stream = GenesysStream(api_base_url="https://dummy.url") inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} expected_params = {"pageSize": 500} assert stream.request_params(**inputs) == expected_params def test_request_headers(patch_base_class): - stream = GenesysStream() + stream = GenesysStream(api_base_url="https://dummy.url") inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} assert len(stream.request_headers(**inputs)) == 0 def test_http_method(patch_base_class): - stream = GenesysStream() + stream = GenesysStream(api_base_url="https://dummy.url") expected_method = "GET" assert stream.http_method == expected_method @@ -48,12 +48,18 @@ def test_http_method(patch_base_class): def test_should_retry(patch_base_class, http_status, should_retry): response_mock = MagicMock() response_mock.status_code = http_status - stream = GenesysStream() + stream = GenesysStream(api_base_url="https://dummy.url") assert stream.should_retry(response_mock) == should_retry def test_backoff_time(patch_base_class): response_mock = MagicMock() - stream = GenesysStream() + stream = GenesysStream(api_base_url="https://dummy.url") expected_backoff_time = 1 assert stream.backoff_time(response_mock) == expected_backoff_time + +def test_url_base(patch_base_class): + api_base_url = "https://dummy.url" + stream = GenesysStream(api_base_url=api_base_url) + assert stream.url_base == api_base_url + "/api/v2/" + From 1d57576a2d9ccbe73578d8983b42ad842c63331d Mon Sep 17 00:00:00 2001 From: Shishir Verma Date: Wed, 26 Apr 2023 12:30:40 +0530 Subject: [PATCH 2/4] update unit tests --- .../source-genesys/source_genesys/source.py | 3 +- .../source-genesys/unit_tests/test_source.py | 31 +++++++++++++++++-- 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/airbyte-integrations/connectors/source-genesys/source_genesys/source.py b/airbyte-integrations/connectors/source-genesys/source_genesys/source.py index e4e4dba98432..9e22a21877d4 100644 --- a/airbyte-integrations/connectors/source-genesys/source_genesys/source.py +++ b/airbyte-integrations/connectors/source-genesys/source_genesys/source.py @@ -18,7 +18,8 @@ class GenesysStream(HttpStream, ABC): page_size = 500 def __init__(self, api_base_url, *args, **kwargs): - self.url_base = api_base_url + "/api/v2/" + if api_base_url is not None: + self.url_base = api_base_url + "/api/v2/" super().__init__(*args, **kwargs) def backoff_time(self, response: requests.Response) -> Optional[int]: diff --git a/airbyte-integrations/connectors/source-genesys/unit_tests/test_source.py b/airbyte-integrations/connectors/source-genesys/unit_tests/test_source.py index d46cbd327756..90e5e1d49972 100644 --- a/airbyte-integrations/connectors/source-genesys/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-genesys/unit_tests/test_source.py @@ -5,6 +5,7 @@ from unittest.mock import MagicMock from source_genesys.source import SourceGenesys +import pytest def test_check_connection(mocker): @@ -17,12 +18,36 @@ def test_check_connection(mocker): def test_streams(mocker): source = SourceGenesys() config_mock = MagicMock() - config_mock.__getitem__.side_effect = lambda key: "Americas (US East)" if key == "tenant_endpoint" else None SourceGenesys.get_connection_response = MagicMock() streams = source.streams(config_mock) expected_streams_number = 16 assert len(streams) == expected_streams_number - expected_url_base = "https://api.mypurecloud.com/api/v2/" + +@pytest.mark.parametrize( + ("tenant_endpoint", "url_base"), + [ + ("Americas (US East)", "https://api.mypurecloud.com/api/v2/"), + ("Americas (US East 2)", "https://api.use2.us-gov-pure.cloud/api/v2/"), + ("Americas (US West)", "https://api.usw2.pure.cloud/api/v2/"), + ("Americas (Canada)", "https://api.cac1.pure.cloud/api/v2/"), + ("Americas (São Paulo)", "https://api.sae1.pure.cloud/api/v2/"), + ("EMEA (Frankfurt)", "https://api.mypurecloud.de/api/v2/"), + ("EMEA (Dublin)", "https://api.mypurecloud.ie/api/v2/"), + ("EMEA (London)", "https://api.euw2.pure.cloud/api/v2/"), + ("Asia Pacific (Mumbai)", "https://api.aps1.pure.cloud/api/v2/"), + ("Asia Pacific (Seoul)", "https://api.apne2.pure.cloud/api/v2/"), + ("Asia Pacific (Sydney)", "https://api.mypurecloud.com.au/api/v2/"), + ], +) +def test_url_base(tenant_endpoint, url_base): + source = SourceGenesys() + config_mock = MagicMock() + config_mock.__getitem__.side_effect = lambda key: tenant_endpoint if key == "tenant_endpoint" else None + SourceGenesys.get_connection_response = MagicMock() + streams = source.streams(config_mock) + expected_streams_number = 16 + assert len(streams) == expected_streams_number + for stream in streams: - assert stream.url_base == expected_url_base + assert stream.url_base == url_base From ef05df356298b5507967989a311a3c2cd7b1f6d2 Mon Sep 17 00:00:00 2001 From: Shishir Verma Date: Thu, 27 Apr 2023 08:44:31 +0530 Subject: [PATCH 3/4] bump up version --- airbyte-integrations/connectors/source-genesys/Dockerfile | 2 +- docs/integrations/sources/genesys.md | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-genesys/Dockerfile b/airbyte-integrations/connectors/source-genesys/Dockerfile index 39db0db35fd9..62b8144a1cea 100644 --- a/airbyte-integrations/connectors/source-genesys/Dockerfile +++ b/airbyte-integrations/connectors/source-genesys/Dockerfile @@ -34,5 +34,5 @@ COPY source_genesys ./source_genesys ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.version=0.1.1 LABEL io.airbyte.name=airbyte/source-genesys diff --git a/docs/integrations/sources/genesys.md b/docs/integrations/sources/genesys.md index e756d2dd2688..36b66946e13c 100644 --- a/docs/integrations/sources/genesys.md +++ b/docs/integrations/sources/genesys.md @@ -24,4 +24,5 @@ You can follow the documentation on [API credentials](https://developer.genesys. ## Changelog | Version | Date | Pull Request | Subject | | :------ | :--------- | :------------------------------------------------------- | :-------------------------- | +| 0.1.1 | 2023-04-27 | [25598](https://github.com/airbytehq/airbyte/pull/25598) | Use region specific API server | | 0.1.0 | 2022-10-06 | [17559](https://github.com/airbytehq/airbyte/pull/17559) | The Genesys Source is created | From 1d8722800037e6f9ec5c1e5eb93aecff951b8d67 Mon Sep 17 00:00:00 2001 From: Shishir Verma Date: Thu, 27 Apr 2023 23:56:25 +0530 Subject: [PATCH 4/4] refactor --- .../source-genesys/source_genesys/source.py | 54 ++++++++----------- 1 file changed, 22 insertions(+), 32 deletions(-) diff --git a/airbyte-integrations/connectors/source-genesys/source_genesys/source.py b/airbyte-integrations/connectors/source-genesys/source_genesys/source.py index 9e22a21877d4..7260e53f2969 100644 --- a/airbyte-integrations/connectors/source-genesys/source_genesys/source.py +++ b/airbyte-integrations/connectors/source-genesys/source_genesys/source.py @@ -14,12 +14,16 @@ class GenesysStream(HttpStream, ABC): - url_base = "https://api.mypurecloud.com.au/api/v2/" page_size = 500 + @property + def url_base(self): + if self._api_base_url is not None: + return self._api_base_url + "/api/v2/" + return None + def __init__(self, api_base_url, *args, **kwargs): - if api_base_url is not None: - self.url_base = api_base_url + "/api/v2/" + self._api_base_url = api_base_url super().__init__(*args, **kwargs) def backoff_time(self, response: requests.Response) -> Optional[int]: @@ -257,36 +261,22 @@ def check_connection(self, logger, config) -> Tuple[bool, any]: def streams(self, config: Mapping[str, Any]) -> List[Stream]: - GENESYS_TENANT_ENDPOINT_MAP: Dict = { - "Americas (US East)": "https://login.mypurecloud.com", - "Americas (US East 2)": "https://login.use2.us-gov-pure.cloud", - "Americas (US West)": "https://login.usw2.pure.cloud", - "Americas (Canada)": "https://login.cac1.pure.cloud", - "Americas (São Paulo)": "https://login.sae1.pure.cloud", - "EMEA (Frankfurt)": "https://login.mypurecloud.de", - "EMEA (Dublin)": "https://login.mypurecloud.ie", - "EMEA (London)": "https://login.euw2.pure.cloud", - "Asia Pacific (Mumbai)": "https://login.aps1.pure.cloud", - "Asia Pacific (Seoul)": "https://login.apne2.pure.cloud", - "Asia Pacific (Sydney)": "https://login.mypurecloud.com.au", - } - base_url = GENESYS_TENANT_ENDPOINT_MAP.get(config["tenant_endpoint"]) - - GENESYS_API_SERVER_MAP: Dict = { - "Americas (US East)": "https://api.mypurecloud.com", - "Americas (US East 2)": "https://api.use2.us-gov-pure.cloud", - "Americas (US West)": "https://api.usw2.pure.cloud", - "Americas (Canada)": "https://api.cac1.pure.cloud", - "Americas (São Paulo)": "https://api.sae1.pure.cloud", - "EMEA (Frankfurt)": "https://api.mypurecloud.de", - "EMEA (Dublin)": "https://api.mypurecloud.ie", - "EMEA (London)": "https://api.euw2.pure.cloud", - "Asia Pacific (Mumbai)": "https://api.aps1.pure.cloud", - "Asia Pacific (Seoul)": "https://api.apne2.pure.cloud", - "Asia Pacific (Sydney)": "https://api.mypurecloud.com.au", + GENESYS_REGION_DOMAIN_MAP: Dict[str, str] = { + "Americas (US East)": "mypurecloud.com", + "Americas (US East 2)": "use2.us-gov-pure.cloud", + "Americas (US West)": "usw2.pure.cloud", + "Americas (Canada)": "cac1.pure.cloud", + "Americas (São Paulo)": "sae1.pure.cloud", + "EMEA (Frankfurt)": "mypurecloud.de", + "EMEA (Dublin)": "mypurecloud.ie", + "EMEA (London)": "euw2.pure.cloud", + "Asia Pacific (Mumbai)": "aps1.pure.cloud", + "Asia Pacific (Seoul)": "apne2.pure.cloud", + "Asia Pacific (Sydney)": "mypurecloud.com.au", } - - api_base_url = GENESYS_API_SERVER_MAP.get(config["tenant_endpoint"]) + domain = GENESYS_REGION_DOMAIN_MAP.get(config["tenant_endpoint"]) + base_url = f"https://login.{domain}" + api_base_url = f"https://api.{domain}" args = {"api_base_url": api_base_url, "authenticator": GenesysOAuthAuthenticator( base_url, config["client_id"], config["client_secret"])}