From 2eb1d2a3189bfe65dcbd545fb9057e80f38cbed7 Mon Sep 17 00:00:00 2001 From: Baz Date: Thu, 4 Aug 2022 13:05:10 +0300 Subject: [PATCH] =?UTF-8?q?=F0=9F=90=9B=20Source=20Zendesk=20Support:=20ad?= =?UTF-8?q?d=20`Subscription=20Plan`=20check=20for=20available=20streams?= =?UTF-8?q?=20(#15233)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../resources/seed/source_definitions.yaml | 2 +- .../src/main/resources/seed/source_specs.yaml | 2 +- .../source-zendesk-support/Dockerfile | 2 +- .../source_zendesk_support/source.py | 55 ++++++++++++------- .../source_zendesk_support/streams.py | 24 +++++++- .../unit_tests/unit_test.py | 27 +++++++-- docs/integrations/sources/zendesk-support.md | 53 +++++++++--------- 7 files changed, 109 insertions(+), 56 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index e909e8c179c6..2a27f7c9e366 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -1065,7 +1065,7 @@ - name: Zendesk Support sourceDefinitionId: 79c1aa37-dae3-42ae-b333-d1c105477715 dockerRepository: airbyte/source-zendesk-support - dockerImageTag: 0.2.14 + dockerImageTag: 0.2.15 documentationUrl: https://docs.airbyte.io/integrations/sources/zendesk-support icon: zendesk.svg sourceType: api diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 70479556b0f0..eb81b025565e 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -10381,7 +10381,7 @@ path_in_connector_config: - "credentials" - "client_secret" -- dockerImage: "airbyte/source-zendesk-support:0.2.14" +- dockerImage: "airbyte/source-zendesk-support:0.2.15" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/zendesk-support" connectionSpecification: diff --git a/airbyte-integrations/connectors/source-zendesk-support/Dockerfile b/airbyte-integrations/connectors/source-zendesk-support/Dockerfile index a9071f0bcf58..76d0301dc991 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/Dockerfile +++ b/airbyte-integrations/connectors/source-zendesk-support/Dockerfile @@ -25,5 +25,5 @@ COPY source_zendesk_support ./source_zendesk_support ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.2.14 +LABEL io.airbyte.version=0.2.15 LABEL io.airbyte.name=airbyte/source-zendesk-support diff --git a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/source.py b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/source.py index 28bda633611d..2213c40c2dca 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/source.py +++ b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/source.py @@ -31,8 +31,13 @@ Tickets, Users, UserSettingsStream, + UserSubscriptionStream, ) +# The Zendesk Subscription Plan gains complete access to all the streams +FULL_ACCESS_PLAN = "Enterprise" +FULL_ACCESS_ONLY_STREAMS = ["ticket_forms"] + class BasicApiTokenAuthenticator(TokenAuthenticator): """basic Authorization header""" @@ -104,24 +109,32 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: :param config: A Mapping of the user input configuration as defined in the connector spec. """ args = self.convert_config2stream_args(config) - # sorted in alphabet order - return [ - GroupMemberships(**args), - Groups(**args), - Macros(**args), - Organizations(**args), - SatisfactionRatings(**args), - SlaPolicies(**args), - Tags(**args), - TicketAudits(**args), - TicketComments(**args), - TicketFields(**args), - TicketForms(**args), - TicketMetrics(**args), - TicketMetricEvents(**args), - Tickets(**args), - Users(**args), - Brands(**args), - CustomRoles(**args), - Schedules(**args), - ] + all_streams_mapping = { + # sorted in alphabet order + "group_membership": GroupMemberships(**args), + "groups": Groups(**args), + "macros": Macros(**args), + "organizations": Organizations(**args), + "satisfaction_ratings": SatisfactionRatings(**args), + "sla_policies": SlaPolicies(**args), + "tags": Tags(**args), + "ticket_audits": TicketAudits(**args), + "ticket_comments": TicketComments(**args), + "ticket_fields": TicketFields(**args), + "ticket_forms": TicketForms(**args), + "ticket_metrics": TicketMetrics(**args), + "ticket_metric_events": TicketMetricEvents(**args), + "tickets": Tickets(**args), + "users": Users(**args), + "brands": Brands(**args), + "custom_roles": CustomRoles(**args), + "schedules": Schedules(**args), + } + # check the users Zendesk Subscription Plan + subscription_plan = UserSubscriptionStream(**args).get_subscription_plan() + if subscription_plan != FULL_ACCESS_PLAN: + # only those the streams that are not listed in FULL_ACCESS_ONLY_STREAMS should be available + return [stream_cls for stream_name, stream_cls in all_streams_mapping.items() if stream_name not in FULL_ACCESS_ONLY_STREAMS] + else: + # all streams should be available for user, otherwise + return [stream_cls for stream_cls in all_streams_mapping.values()] diff --git a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/streams.py b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/streams.py index acc9c217174a..30bfa7269756 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/streams.py +++ b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/streams.py @@ -47,7 +47,7 @@ class SourceZendeskException(Exception): class SourceZendeskSupportFuturesSession(FuturesSession): """ - Check the docs at https://github.com/ross/requests-futures. + Check the docs at https://github.com/ross/requests-futures Used to async execute a set of requests. """ @@ -534,7 +534,7 @@ class TicketFields(SourceZendeskSupportStream): class TicketForms(SourceZendeskSupportCursorPaginationStream): - """TicketForms stream: https://developer.zendesk.com/api-reference/ticketing/tickets/ticket_forms/""" + """TicketForms stream: https://developer.zendesk.com/api-reference/ticketing/tickets/ticket_forms""" class TicketMetrics(SourceZendeskSupportCursorPaginationStream): @@ -646,3 +646,23 @@ def get_settings(self) -> Mapping[str, Any]: for resp in self.read_records(SyncMode.full_refresh): return resp raise SourceZendeskException("not found settings") + + +class UserSubscriptionStream(SourceZendeskSupportFullRefreshStream): + """Stream for checking read permissions for streams""" + + def path(self, *args, **kwargs) -> str: + return "/api/v2/account/subscription.json" + + def next_page_token(self, *args, **kwargs) -> Optional[Mapping[str, Any]]: + return None + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + subscription_plan = response.json().get("subscription").get("plan_name") + if subscription_plan: + yield subscription_plan + + def get_subscription_plan(self) -> Mapping[str, Any]: + for result in self.read_records(SyncMode.full_refresh): + return result + raise SourceZendeskException("Could not read User's Subscription Plan.") diff --git a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/unit_test.py index 37981508f170..3fb7c0f31e88 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/unit_test.py @@ -40,6 +40,7 @@ Tickets, Users, UserSettingsStream, + UserSubscriptionStream, ) from test_data.data import TICKET_EVENTS_STREAM_RESPONSE from utils import read_full_refresh @@ -130,6 +131,22 @@ def test_check(response, check_passed): assert check_passed == result +@pytest.mark.parametrize( + "response, expected_n_streams", + [ + ("Enterprise", 18), + # if restricted, TicketForms stream will not be listed + ("Other", 17), + ], + ids=["full_access", "restricted_access"], +) +def test_full_access_streams(response, expected_n_streams): + with patch.object(UserSubscriptionStream, "get_subscription_plan", return_value=response) as mock_method: + result = SourceZendeskSupport().streams(config=TEST_CONFIG) + mock_method.assert_called() + assert len(result) == expected_n_streams + + @pytest.fixture(autouse=True) def time_sleep_mock(mocker): time_mock = mocker.patch("time.sleep", lambda x: None) @@ -265,10 +282,12 @@ class TestAllStreams: ], ) def test_streams(self, expected_stream_cls): - streams = SourceZendeskSupport().streams(TEST_CONFIG) - for stream in streams: - if expected_stream_cls in streams: - assert isinstance(stream, expected_stream_cls) + with patch.object(UserSubscriptionStream, "get_subscription_plan", return_value="Enterprise") as mock_method: + streams = SourceZendeskSupport().streams(TEST_CONFIG) + mock_method.assert_called() + for stream in streams: + if expected_stream_cls in streams: + assert isinstance(stream, expected_stream_cls) @pytest.mark.parametrize( "stream_cls, expected", diff --git a/docs/integrations/sources/zendesk-support.md b/docs/integrations/sources/zendesk-support.md index f279eb8b2896..b7d4c7eede3b 100644 --- a/docs/integrations/sources/zendesk-support.md +++ b/docs/integrations/sources/zendesk-support.md @@ -60,30 +60,31 @@ The Zendesk connector ideally should not run into Zendesk API limitations under | Version | Date | Pull Request | Subject | |:---------|:-----------|:---------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `0.2.14` | 2022-07-27 | [15036](https://github.com/airbytehq/airbyte/pull/15036) | Convert `ticket_audits.previous_value` values to string | -| `0.2.13` | 2022-07-21 | [14829](https://github.com/airbytehq/airbyte/pull/14829) | Convert `tickets.custom_fields` values to string | -| `0.2.12` | 2022-06-30 | [14304](https://github.com/airbytehq/airbyte/pull/14304) | Fixed Pagination for Group Membership stream | -| `0.2.11` | 2022-06-24 | [14112](https://github.com/airbytehq/airbyte/pull/14112) | Fixed "Retry-After" non integer value | -| `0.2.10` | 2022-06-14 | [13757](https://github.com/airbytehq/airbyte/pull/13757) | Fixed the bug with `TicketMetrics` stream, HTTP Error 429, caused by lots of API requests | -| `0.2.9` | 2022-05-27 | [13261](https://github.com/airbytehq/airbyte/pull/13261) | Bugfix for the unhandled [ChunkedEncodingError](https://github.com/airbytehq/airbyte/issues/12591) and [ConnectionError](https://github.com/airbytehq/airbyte/issues/12155) | -| `0.2.8` | 2022-05-20 | [13055](https://github.com/airbytehq/airbyte/pull/13055) | Fixed minor issue for stream `ticket_audits` schema | -| `0.2.7` | 2022-04-27 | [12335](https://github.com/airbytehq/airbyte/pull/12335) | Adding fixtures to mock time.sleep for connectors that explicitly sleep | +| `0.2.15` | 2022-08-03 | [15233](https://github.com/airbytehq/airbyte/pull/15233) | Added `subscription plan` check on `streams discovery` step to remove streams that are not accessible for fetch due to subscription plan restrictions | +| `0.2.14` | 2022-07-27 | [15036](https://github.com/airbytehq/airbyte/pull/15036) | Convert `ticket_audits.previous_value` values to string | +| `0.2.13` | 2022-07-21 | [14829](https://github.com/airbytehq/airbyte/pull/14829) | Convert `tickets.custom_fields` values to string | +| `0.2.12` | 2022-06-30 | [14304](https://github.com/airbytehq/airbyte/pull/14304) | Fixed Pagination for Group Membership stream | +| `0.2.11` | 2022-06-24 | [14112](https://github.com/airbytehq/airbyte/pull/14112) | Fixed "Retry-After" non integer value | +| `0.2.10` | 2022-06-14 | [13757](https://github.com/airbytehq/airbyte/pull/13757) | Fixed the bug with `TicketMetrics` stream, HTTP Error 429, caused by lots of API requests | +| `0.2.9` | 2022-05-27 | [13261](https://github.com/airbytehq/airbyte/pull/13261) | Bugfix for the unhandled [ChunkedEncodingError](https://github.com/airbytehq/airbyte/issues/12591) and [ConnectionError](https://github.com/airbytehq/airbyte/issues/12155)| +| `0.2.8` | 2022-05-20 | [13055](https://github.com/airbytehq/airbyte/pull/13055) | Fixed minor issue for stream `ticket_audits` schema | +| `0.2.7` | 2022-04-27 | [12335](https://github.com/airbytehq/airbyte/pull/12335) | Adding fixtures to mock time.sleep for connectors that explicitly sleep | | `0.2.6` | 2022-04-19 | [12122](https://github.com/airbytehq/airbyte/pull/12122) | Fixed the bug when only 100,000 Users are synced [11895](https://github.com/airbytehq/airbyte/issues/11895) and fixed bug when `start_date` is not used on user stream [12059](https://github.com/airbytehq/airbyte/issues/12059). | -| `0.2.5` | 2022-04-05 | [11727](https://github.com/airbytehq/airbyte/pull/11727) | Fixed the bug when state was not parsed correctly | -| `0.2.4` | 2022-04-04 | [11688](https://github.com/airbytehq/airbyte/pull/11688) | Small documentation corrections | -| `0.2.3` | 2022-03-23 | [11349](https://github.com/airbytehq/airbyte/pull/11349) | Fixed the bug when Tickets stream didn't return deleted records | -| `0.2.2` | 2022-03-17 | [11237](https://github.com/airbytehq/airbyte/pull/11237) | Fixed the bug when TicketComments stream didn't return all records | -| `0.2.1` | 2022-03-15 | [11162](https://github.com/airbytehq/airbyte/pull/11162) | Added support of OAuth2.0 authentication method | -| `0.2.0` | 2022-03-01 | [9456](https://github.com/airbytehq/airbyte/pull/9456) | Update source to use future requests | -| `0.1.12` | 2022-01-25 | [9785](https://github.com/airbytehq/airbyte/pull/9785) | Add additional log messages | -| `0.1.11` | 2021-12-21 | [8987](https://github.com/airbytehq/airbyte/pull/8987) | Update connector fields title/description | -| `0.1.9` | 2021-12-16 | [8616](https://github.com/airbytehq/airbyte/pull/8616) | Adds Brands, CustomRoles and Schedules streams | -| `0.1.8` | 2021-11-23 | [8050](https://github.com/airbytehq/airbyte/pull/8168) | Adds TicketMetricEvents stream | -| `0.1.7` | 2021-11-23 | [8058](https://github.com/airbytehq/airbyte/pull/8058) | Added support of AccessToken authentication | -| `0.1.6` | 2021-11-18 | [8050](https://github.com/airbytehq/airbyte/pull/8050) | Fix wrong types for schemas, add TypeTransformer | -| `0.1.5` | 2021-10-26 | [7679](https://github.com/airbytehq/airbyte/pull/7679) | Add ticket_id and ticket_comments | -| `0.1.4` | 2021-10-26 | [7377](https://github.com/airbytehq/airbyte/pull/7377) | Fix initially_assigned_at type in ticket metrics | -| `0.1.3` | 2021-10-17 | [7097](https://github.com/airbytehq/airbyte/pull/7097) | Corrected the connector's specification | -| `0.1.2` | 2021-10-16 | [6513](https://github.com/airbytehq/airbyte/pull/6513) | Fixed TicketComments stream | -| `0.1.1` | 2021-09-02 | [5787](https://github.com/airbytehq/airbyte/pull/5787) | Fixed incremental logic for the ticket_comments stream | -| `0.1.0` | 2021-07-21 | [4861](https://github.com/airbytehq/airbyte/pull/4861) | Created CDK native zendesk connector | +| `0.2.5` | 2022-04-05 | [11727](https://github.com/airbytehq/airbyte/pull/11727) | Fixed the bug when state was not parsed correctly | +| `0.2.4` | 2022-04-04 | [11688](https://github.com/airbytehq/airbyte/pull/11688) | Small documentation corrections | +| `0.2.3` | 2022-03-23 | [11349](https://github.com/airbytehq/airbyte/pull/11349) | Fixed the bug when Tickets stream didn't return deleted records | +| `0.2.2` | 2022-03-17 | [11237](https://github.com/airbytehq/airbyte/pull/11237) | Fixed the bug when TicketComments stream didn't return all records | +| `0.2.1` | 2022-03-15 | [11162](https://github.com/airbytehq/airbyte/pull/11162) | Added support of OAuth2.0 authentication method | +| `0.2.0` | 2022-03-01 | [9456](https://github.com/airbytehq/airbyte/pull/9456) | Update source to use future requests | +| `0.1.12` | 2022-01-25 | [9785](https://github.com/airbytehq/airbyte/pull/9785) | Add additional log messages | +| `0.1.11` | 2021-12-21 | [8987](https://github.com/airbytehq/airbyte/pull/8987) | Update connector fields title/description | +| `0.1.9` | 2021-12-16 | [8616](https://github.com/airbytehq/airbyte/pull/8616) | Adds Brands, CustomRoles and Schedules streams | +| `0.1.8` | 2021-11-23 | [8050](https://github.com/airbytehq/airbyte/pull/8168) | Adds TicketMetricEvents stream | +| `0.1.7` | 2021-11-23 | [8058](https://github.com/airbytehq/airbyte/pull/8058) | Added support of AccessToken authentication | +| `0.1.6` | 2021-11-18 | [8050](https://github.com/airbytehq/airbyte/pull/8050) | Fix wrong types for schemas, add TypeTransformer | +| `0.1.5` | 2021-10-26 | [7679](https://github.com/airbytehq/airbyte/pull/7679) | Add ticket_id and ticket_comments | +| `0.1.4` | 2021-10-26 | [7377](https://github.com/airbytehq/airbyte/pull/7377) | Fix initially_assigned_at type in ticket metrics | +| `0.1.3` | 2021-10-17 | [7097](https://github.com/airbytehq/airbyte/pull/7097) | Corrected the connector's specification | +| `0.1.2` | 2021-10-16 | [6513](https://github.com/airbytehq/airbyte/pull/6513) | Fixed TicketComments stream | +| `0.1.1` | 2021-09-02 | [5787](https://github.com/airbytehq/airbyte/pull/5787) | Fixed incremental logic for the ticket_comments stream | +| `0.1.0` | 2021-07-21 | [4861](https://github.com/airbytehq/airbyte/pull/4861) | Created CDK native zendesk connector |