Source Zendesk Support: Fix cursor pagination (#28487)

* Source Zendesk Support: Remove extra page from params

* Source Zendesk Support: bump version

* Source Zendesk Support: Refactor + Fix pagination issues

* Source Zendesk Support: fix object count

* Source Zendesk Support: update timeout for tests

* Source Zendesk Support: update expected_records

* Source Zendesk Support: update expected_records

* Source Zendesk Support: update ignore pagination

airbyte-integrations/connectors/source-zendesk-support/Dockerfile

@@ -25,5 +25,5 @@ COPY source_zendesk_support ./source_zendesk_support
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.10.1
+LABEL io.airbyte.version=0.10.2
 LABEL io.airbyte.name=airbyte/source-zendesk-support

airbyte-integrations/connectors/source-zendesk-support/acceptance-test-config.yml

@@ -21,6 +21,7 @@ acceptance_tests:
   basic_read:
     tests:
       - config_path: "secrets/config.json"
+        timeout_seconds: 2400
         expect_records:
           path: "integration_tests/expected_records.jsonl"
           extra_fields: no
@@ -49,3 +50,4 @@ acceptance_tests:
     tests:
       - config_path: "secrets/config.json"
         configured_catalog_path: "integration_tests/configured_catalog.json"
+        timeout_seconds: 2400

airbyte-integrations/connectors/source-zendesk-support/integration_tests/expected_records.jsonl

@@ -29,9 +29,9 @@
 {"stream":"ticket_fields","data":{"url":"https://d3v-airbyte.zendesk.com/api/v2/ticket_fields/360002833096.json","id":360002833096,"type":"description","title":"Description","raw_title":"Description","description":"Please enter the details of your request. A member of our support staff will respond as soon as possible.","raw_description":"Please enter the details of your request. A member of our support staff will respond as soon as possible.","position":2,"active":true,"required":false,"collapsed_for_agents":false,"regexp_for_validation":null,"title_in_portal":"Description","raw_title_in_portal":"Description","visible_in_portal":true,"editable_in_portal":true,"required_in_portal":true,"tag":null,"created_at":"2020-12-11T18:34:05Z","updated_at":"2020-12-11T18:34:05Z","removable":false,"key":null,"agent_description":null},"emitted_at":1687861693520}
 {"stream":"ticket_fields","data":{"url":"https://d3v-airbyte.zendesk.com/api/v2/ticket_fields/360002833116.json","id":360002833116,"type":"status","title":"Status","raw_title":"Status","description":"Request status","raw_description":"Request status","position":3,"active":true,"required":false,"collapsed_for_agents":false,"regexp_for_validation":null,"title_in_portal":"Status","raw_title_in_portal":"Status","visible_in_portal":false,"editable_in_portal":false,"required_in_portal":false,"tag":null,"created_at":"2020-12-11T18:34:05Z","updated_at":"2020-12-11T18:34:05Z","removable":false,"key":null,"agent_description":null,"system_field_options":[{"name":"Open","value":"open"},{"name":"Pending","value":"pending"},{"name":"Solved","value":"solved"}],"sub_type_id":0},"emitted_at":1687861693521}
 {"stream":"ticket_forms","data":{"url":"https://d3v-airbyte.zendesk.com/api/v2/ticket_forms/360000084116.json","name":"Default Ticket Form","display_name":"Default Ticket Form","id":360000084116,"raw_name":"Default Ticket Form","raw_display_name":"Default Ticket Form","end_user_visible":true,"position":1,"ticket_field_ids":[360002833076,360002833096,360002833116,360002833136,360002833156,360002833176,360002833196],"active":true,"default":true,"created_at":"2020-12-11T18:34:37Z","updated_at":"2020-12-11T18:34:37Z","in_all_brands":true,"restricted_brand_ids":[],"end_user_conditions":[],"agent_conditions":[]},"emitted_at":1687861694440}
-{"stream":"ticket_metrics","data":{"url":"https://d3v-airbyte.zendesk.com/api/v2/ticket_metrics/7283000498191.json","id":7283000498191,"ticket_id":153,"created_at":"2023-06-26T11:31:48Z","updated_at":"2023-06-26T12:13:42Z","group_stations":2,"assignee_stations":2,"reopens":0,"replies":0,"assignee_updated_at":"2023-06-26T11:31:48Z","requester_updated_at":"2023-06-26T11:31:48Z","status_updated_at":"2023-06-26T11:31:48Z","initially_assigned_at":"2023-06-26T11:31:48Z","assigned_at":"2023-06-26T12:13:42Z","solved_at":null,"latest_comment_added_at":"2023-06-26T11:31:48Z","reply_time_in_minutes":{"calendar":null,"business":null},"first_resolution_time_in_minutes":{"calendar":null,"business":null},"full_resolution_time_in_minutes":{"calendar":null,"business":null},"agent_wait_time_in_minutes":{"calendar":null,"business":null},"requester_wait_time_in_minutes":{"calendar":null,"business":null},"on_hold_time_in_minutes":{"calendar":0,"business":0},"custom_status_updated_at":"2023-06-26T11:31:48Z"},"emitted_at":1687861695566}
-{"stream":"ticket_metrics","data":{"url":"https://d3v-airbyte.zendesk.com/api/v2/ticket_metrics/7282909551759.json","id":7282909551759,"ticket_id":152,"created_at":"2023-06-26T11:10:33Z","updated_at":"2023-06-26T11:25:43Z","group_stations":1,"assignee_stations":1,"reopens":0,"replies":1,"assignee_updated_at":"2023-06-26T11:25:43Z","requester_updated_at":"2023-06-26T11:10:33Z","status_updated_at":"2023-06-26T11:25:43Z","initially_assigned_at":"2023-06-26T11:10:33Z","assigned_at":"2023-06-26T11:10:33Z","solved_at":"2023-06-26T11:25:43Z","latest_comment_added_at":"2023-06-26T11:21:06Z","reply_time_in_minutes":{"calendar":11,"business":0},"first_resolution_time_in_minutes":{"calendar":15,"business":0},"full_resolution_time_in_minutes":{"calendar":15,"business":0},"agent_wait_time_in_minutes":{"calendar":15,"business":0},"requester_wait_time_in_minutes":{"calendar":0,"business":0},"on_hold_time_in_minutes":{"calendar":0,"business":0},"custom_status_updated_at":"2023-06-26T11:25:43Z"},"emitted_at":1687861695567}
-{"stream":"ticket_metrics","data":{"url":"https://d3v-airbyte.zendesk.com/api/v2/ticket_metrics/7282901696015.json","id":7282901696015,"ticket_id":151,"created_at":"2023-06-26T11:09:33Z","updated_at":"2023-06-26T12:03:38Z","group_stations":1,"assignee_stations":1,"reopens":0,"replies":1,"assignee_updated_at":"2023-06-26T12:03:37Z","requester_updated_at":"2023-06-26T11:09:33Z","status_updated_at":"2023-06-26T11:09:33Z","initially_assigned_at":"2023-06-26T11:09:33Z","assigned_at":"2023-06-26T11:09:33Z","solved_at":null,"latest_comment_added_at":"2023-06-26T12:03:37Z","reply_time_in_minutes":{"calendar":54,"business":0},"first_resolution_time_in_minutes":{"calendar":null,"business":null},"full_resolution_time_in_minutes":{"calendar":null,"business":null},"agent_wait_time_in_minutes":{"calendar":null,"business":null},"requester_wait_time_in_minutes":{"calendar":null,"business":null},"on_hold_time_in_minutes":{"calendar":0,"business":0},"custom_status_updated_at":"2023-06-26T11:09:33Z"},"emitted_at":1687861695567}
+{"stream":"ticket_metrics","data":{"url":"https://d3v-airbyte.zendesk.com/api/v2/ticket_metrics/7283000498191.json","id":7283000498191,"ticket_id":153,"created_at":"2023-06-26T11:31:48Z","updated_at":"2023-06-26T12:13:42Z","group_stations":2,"assignee_stations":2,"reopens":0,"replies":0,"assignee_updated_at":"2023-06-26T11:31:48Z","requester_updated_at":"2023-06-26T11:31:48Z","status_updated_at":"2023-06-26T11:31:48Z","initially_assigned_at":"2023-06-26T11:31:48Z","assigned_at":"2023-06-26T12:13:42Z","solved_at":null,"latest_comment_added_at":"2023-06-26T11:31:48Z","reply_time_in_minutes":{"calendar":null,"business":null},"first_resolution_time_in_minutes":{"calendar":null,"business":null},"full_resolution_time_in_minutes":{"calendar":null,"business":null},"agent_wait_time_in_minutes":{"calendar":null,"business":null},"requester_wait_time_in_minutes":{"calendar":null,"business":null},"on_hold_time_in_minutes":{"calendar":0,"business":0},"custom_status_updated_at":"2023-06-26T11:31:48Z"},"emitted_at":1689884373175}
+{"stream":"ticket_metrics","data":{"url":"https://d3v-airbyte.zendesk.com/api/v2/ticket_metrics/7282909551759.json","id":7282909551759,"ticket_id":152,"created_at":"2023-06-26T11:10:33Z","updated_at":"2023-06-26T11:25:43Z","group_stations":1,"assignee_stations":1,"reopens":0,"replies":1,"assignee_updated_at":"2023-06-26T11:25:43Z","requester_updated_at":"2023-06-26T11:10:33Z","status_updated_at":"2023-07-16T12:01:39Z","initially_assigned_at":"2023-06-26T11:10:33Z","assigned_at":"2023-06-26T11:10:33Z","solved_at":"2023-06-26T11:25:43Z","latest_comment_added_at":"2023-06-26T11:21:06Z","reply_time_in_minutes":{"calendar":11,"business":0},"first_resolution_time_in_minutes":{"calendar":15,"business":0},"full_resolution_time_in_minutes":{"calendar":15,"business":0},"agent_wait_time_in_minutes":{"calendar":15,"business":0},"requester_wait_time_in_minutes":{"calendar":0,"business":0},"on_hold_time_in_minutes":{"calendar":0,"business":0},"custom_status_updated_at":"2023-06-26T11:25:43Z"},"emitted_at":1689884373175}
+{"stream":"ticket_metrics","data":{"url":"https://d3v-airbyte.zendesk.com/api/v2/ticket_metrics/7282901696015.json","id":7282901696015,"ticket_id":151,"created_at":"2023-06-26T11:09:33Z","updated_at":"2023-06-26T12:03:38Z","group_stations":1,"assignee_stations":1,"reopens":0,"replies":1,"assignee_updated_at":"2023-06-26T12:03:37Z","requester_updated_at":"2023-06-26T11:09:33Z","status_updated_at":"2023-06-26T11:09:33Z","initially_assigned_at":"2023-06-26T11:09:33Z","assigned_at":"2023-06-26T11:09:33Z","solved_at":null,"latest_comment_added_at":"2023-06-26T12:03:37Z","reply_time_in_minutes":{"calendar":54,"business":0},"first_resolution_time_in_minutes":{"calendar":null,"business":null},"full_resolution_time_in_minutes":{"calendar":null,"business":null},"agent_wait_time_in_minutes":{"calendar":null,"business":null},"requester_wait_time_in_minutes":{"calendar":null,"business":null},"on_hold_time_in_minutes":{"calendar":0,"business":0},"custom_status_updated_at":"2023-06-26T11:09:33Z"},"emitted_at":1689884373175}
 {"stream":"ticket_metric_events","data":{"id":4992797383183,"ticket_id":121,"metric":"agent_work_time","instance_id":0,"type":"measure","time":"2022-06-17T14:49:20Z"},"emitted_at":1687861699258}
 {"stream":"ticket_metric_events","data":{"id":4992797383311,"ticket_id":121,"metric":"pausable_update_time","instance_id":0,"type":"measure","time":"2022-06-17T14:49:20Z"},"emitted_at":1687861699259}
 {"stream":"ticket_metric_events","data":{"id":4992797383439,"ticket_id":121,"metric":"reply_time","instance_id":0,"type":"measure","time":"2022-06-17T14:49:20Z"},"emitted_at":1687861699260}
@@ -52,4 +52,4 @@
 {"stream": "attribute_definitions", "data": {"title": "Form", "subject": "ticket_form_id", "type": "list", "group": "ticket", "nullable": false, "repeatable": false, "operators": [{"value": "is", "title": "Is", "terminal": false}, {"value": "is_not", "title": "Is not", "terminal": false}], "values": [{"value": "360000084116", "title": "Default Ticket Form", "enabled": true}], "condition": "all"}, "emitted_at": 1687777243928}
 {"stream":"ticket_skips","data":{"id":7290033348623,"ticket_id":121,"user_id":360786799676,"reason":"I have no idea.","created_at":"2023-06-27T08:24:02Z","updated_at":"2023-06-27T08:24:02Z","ticket":{"url":"https://d3v-airbyte.zendesk.com/api/v2/tickets/121.json","id":121,"external_id":null,"via":{"channel":"voice","source":{"rel":"voicemail","from":{"formatted_phone":"+1 (689) 689-8023","phone":"+16896898023","name":"Caller +1 (689) 689-8023"},"to":{"formatted_phone":"+1 (205) 953-1462","phone":"+12059531462","name":"Airbyte","brand_id":360000358316}}},"created_at":"2022-06-17T14:49:20Z","updated_at":"2022-06-17T16:01:42Z","type":null,"subject":"Voicemail from: Caller +1 (689) 689-8023","raw_subject":"Voicemail from: Caller +1 (689) 689-8023","description":"Call from: +1 (689) 689-8023\\nTime of call: June 17, 2022 at 2:48:27 PM","priority":null,"status":"new","recipient":null,"requester_id":4992781783439,"submitter_id":4992781783439,"assignee_id":null,"organization_id":null,"group_id":null,"collaborator_ids":[],"follower_ids":[],"email_cc_ids":[],"forum_topic_id":null,"problem_id":null,"has_incidents":false,"is_public":false,"due_at":null,"tags":[],"custom_fields":[],"satisfaction_rating":{"score":"offered"},"sharing_agreement_ids":[],"custom_status_id":4044356,"fields":[],"followup_ids":[],"ticket_form_id":360000084116,"deleted_ticket_form_id":null,"brand_id":360000358316,"allow_channelback":false,"allow_attachments":true,"from_messaging_channel":false}},"emitted_at":1687861697932}
 {"stream":"ticket_skips","data":{"id":7290088475023,"ticket_id":125,"user_id":360786799676,"reason":"Another test skip.","created_at":"2023-06-27T08:30:01Z","updated_at":"2023-06-27T08:30:01Z","ticket":{"url":"https://d3v-airbyte.zendesk.com/api/v2/tickets/125.json","id":125,"external_id":null,"via":{"channel":"web","source":{"from":{},"to":{},"rel":null}},"created_at":"2022-07-18T10:16:53Z","updated_at":"2022-07-18T10:36:02Z","type":"question","subject":"Ticket Test 2","raw_subject":"Ticket Test 2","description":"238473846","priority":"urgent","status":"open","recipient":null,"requester_id":360786799676,"submitter_id":360786799676,"assignee_id":361089721035,"organization_id":360033549136,"group_id":5059439464079,"collaborator_ids":[360786799676],"follower_ids":[360786799676],"email_cc_ids":[],"forum_topic_id":null,"problem_id":null,"has_incidents":false,"is_public":false,"due_at":null,"tags":[],"custom_fields":[],"satisfaction_rating":{"score":"unoffered"},"sharing_agreement_ids":[],"custom_status_id":4044376,"fields":[],"followup_ids":[],"ticket_form_id":360000084116,"deleted_ticket_form_id":null,"brand_id":360000358316,"allow_channelback":false,"allow_attachments":true,"from_messaging_channel":false}},"emitted_at":1687861697934}
-{"stream":"posts","data":{"id":7253375870607,"title":"Which topics should I add to my community?","details":"<p>That depends. If you support several products, you might add a topic for each product. If you have one big product, you might add a topic for each major feature area or task. If you have different types of users (for example, end users and API developers), you might add a topic or topics for each type of user.</p><p>A General Discussion topic is a place for users to discuss issues that don't quite fit in the other topics. You could monitor this topic for emerging issues that might need their own topics.</p>\n\n<p>To create your own topics, see <a href=\"https://support.zendesk.com/hc/en-us/articles/205586568#topic_dqx_znw_3k\" target=\"_blank\" rel=\"nofollow noreferrer\">Adding community discussion topics</a>.</p>","author_id":360786799676,"vote_sum":0,"vote_count":0,"comment_count":0,"follower_count":0,"topic_id":7253351897871,"html_url":"https://d3v-airbyte.zendesk.com/hc/en-us/community/posts/7253375870607-Which-topics-should-I-add-to-my-community-","created_at":"2023-06-22T00:32:21Z","updated_at":"2023-06-22T00:32:21Z","url":"https://d3v-airbyte.zendesk.com/api/v2/help_center/community/posts/7253375870607-Which-topics-should-I-add-to-my-community-.json","featured":false,"pinned":false,"closed":false,"frozen":false,"status":"none","non_author_editor_id":null,"non_author_updated_at":null,"content_tag_ids":[]},"emitted_at":1688581224313}
+{"stream":"posts","data":{"id":7253375870607,"title":"Which topics should I add to my community?","details":"<p>That depends. If you support several products, you might add a topic for each product. If you have one big product, you might add a topic for each major feature area or task. If you have different types of users (for example, end users and API developers), you might add a topic or topics for each type of user.</p><p>A General Discussion topic is a place for users to discuss issues that don't quite fit in the other topics. You could monitor this topic for emerging issues that might need their own topics.</p>\n\n<p>To create your own topics, see <a href=\"https://support.zendesk.com/hc/en-us/articles/205586568#topic_dqx_znw_3k\" target=\"_blank\" rel=\"nofollow noreferrer\">Adding community discussion topics</a>.</p>","author_id":360786799676,"vote_sum":0,"vote_count":0,"comment_count":0,"follower_count":0,"topic_id":7253351897871,"html_url":"https://d3v-airbyte.zendesk.com/hc/en-us/community/posts/7253375870607-Which-topics-should-I-add-to-my-community-","created_at":"2023-06-22T00:32:21Z","updated_at":"2023-06-22T00:32:21Z","url":"https://d3v-airbyte.zendesk.com/api/v2/help_center/community/posts/7253375870607-Which-topics-should-I-add-to-my-community-.json","featured":false,"pinned":false,"closed":false,"frozen":false,"status":"none","non_author_editor_id":null,"non_author_updated_at":null},"emitted_at":1689889045524}

airbyte-integrations/connectors/source-zendesk-support/metadata.yaml

@@ -7,7 +7,7 @@ data:
   connectorType: source
   maxSecondsBetweenMessages: 10800
   definitionId: 79c1aa37-dae3-42ae-b333-d1c105477715
-  dockerImageTag: 0.10.1
+  dockerImageTag: 0.10.2
   dockerRepository: airbyte/source-zendesk-support
   githubIssueLabel: source-zendesk-support
   icon: zendesk-support.svg

airbyte-integrations/connectors/source-zendesk-support/setup.py

@@ -5,7 +5,7 @@
 
 from setuptools import find_packages, setup
 
-MAIN_REQUIREMENTS = ["airbyte-cdk", "pytz", "requests-futures~=1.0.0", "pendulum~=2.1.2"]
+MAIN_REQUIREMENTS = ["airbyte-cdk", "pytz"]
 
 TEST_REQUIREMENTS = ["pytest~=6.1", "pytest-mock~=3.6", "connector-acceptance-test", "requests-mock==1.9.3"]