add mock server test for semi-incremental syncs

airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/helpers.py

@@ -1,15 +1,33 @@
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-from typing import Optional
 
+import pendulum
 from airbyte_cdk.test.mock_http import HttpMocker
 from airbyte_cdk.test.mock_http.response_builder import FieldPath
 from pendulum.datetime import DateTime
 
 from .utils import datetime_to_string
-from .zs_requests import PostsCommentsRequestBuilder, PostsRequestBuilder, TicketFormsRequestBuilder, TicketsRequestBuilder
+from .zs_requests import (
+    GroupsRequestBuilder,
+    PostsCommentsRequestBuilder,
+    PostsRequestBuilder,
+    TicketFormsRequestBuilder,
+    TicketsRequestBuilder,
+)
 from .zs_requests.request_authenticators import ApiTokenAuthenticator
-from .zs_responses import PostsCommentsResponseBuilder, PostsResponseBuilder, TicketFormsResponseBuilder, TicketsResponseBuilder
-from .zs_responses.records import PostsCommentsRecordBuilder, PostsRecordBuilder, TicketFormsRecordBuilder, TicketsRecordBuilder
+from .zs_responses import (
+    GroupsResponseBuilder,
+    PostsCommentsResponseBuilder,
+    PostsResponseBuilder,
+    TicketFormsResponseBuilder,
+    TicketsResponseBuilder,
+)
+from .zs_responses.records import (
+    GroupsRecordBuilder,
+    PostsCommentsRecordBuilder,
+    PostsRecordBuilder,
+    TicketFormsRecordBuilder,
+    TicketsRecordBuilder,
+)
 
 
 def given_ticket_forms(
@@ -90,3 +108,30 @@ def given_tickets_with_state(http_mocker: HttpMocker, start_date: DateTime, curs
         TicketsResponseBuilder.tickets_response().with_record(tickets_record_builder).build(),
     )
     return tickets_record_builder
+
+
+def given_groups_with_later_records(
+    http_mocker: HttpMocker,
+    updated_at_value: DateTime,
+    later_record_time_delta: pendulum.duration,
+    api_token_authenticator: ApiTokenAuthenticator
+) -> GroupsRecordBuilder:
+    """
+    Creates two group records one with a specific cursor value and one that has a later cursor value based on the
+    provided timedelta. This is intended to create multiple records with different times which can be used to
+    test functionality like semi-incremental record filtering
+    """
+    groups_record_builder = GroupsRecordBuilder.groups_record().with_field(
+        FieldPath("updated_at"), datetime_to_string(updated_at_value)
+    )
+
+    later_groups_record_builder = GroupsRecordBuilder.groups_record().with_field(
+        FieldPath("updated_at"), datetime_to_string(updated_at_value + later_record_time_delta)
+    )
+    http_mocker.get(
+        GroupsRequestBuilder.groups_endpoint(api_token_authenticator)
+        .with_page_size(100)
+        .build(),
+        GroupsResponseBuilder.groups_response().with_record(groups_record_builder).with_record(later_groups_record_builder).build(),
+    )
+    return groups_record_builder

airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/test_groups.py

@@ -0,0 +1,72 @@
+# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
+
+from datetime import datetime, timezone
+from unittest import TestCase
+
+import pendulum
+from airbyte_cdk.models import SyncMode
+from airbyte_cdk.test.mock_http import HttpMocker
+from airbyte_cdk.test.state_builder import StateBuilder
+
+from .config import ConfigBuilder
+from .helpers import given_groups_with_later_records
+from .utils import datetime_to_string, read_stream, string_to_datetime
+from .zs_requests.request_authenticators import ApiTokenAuthenticator
+
+_NOW = datetime.now(timezone.utc)
+
+
+class TestGroupsStreamFullRefresh(TestCase):
+    @property
+    def _config(self):
+        return (
+            ConfigBuilder()
+            .with_basic_auth_credentials("user@example.com", "password")
+            .with_subdomain("d3v-airbyte")
+            .with_start_date(pendulum.now(tz="UTC").subtract(years=2))
+            .build()
+        )
+
+    @staticmethod
+    def get_authenticator(config):
+        return ApiTokenAuthenticator(email=config["credentials"]["email"], password=config["credentials"]["api_token"])
+
+    @HttpMocker()
+    def test_given_incoming_state_semi_incremental_groups_does_not_emit_earlier_record(self, http_mocker):
+        """
+        Perform a semi-incremental sync where records that came before the current state are not included in the set
+        of records emitted
+        """
+        api_token_authenticator = self.get_authenticator(self._config)
+        given_groups_with_later_records(
+            http_mocker,
+            string_to_datetime(self._config["start_date"]),
+            pendulum.duration(weeks=12),
+            api_token_authenticator,
+        )
+
+        output = read_stream("groups", SyncMode.full_refresh, self._config)
+        assert len(output.records) == 2
+
+    @HttpMocker()
+    def test_given_incoming_state_semi_incremental_groups_does_not_emit_earlier_record(self, http_mocker):
+        """
+        Perform a semi-incremental sync where records that came before the current state are not included in the set
+        of records emitted
+        """
+        api_token_authenticator = self.get_authenticator(self._config)
+        given_groups_with_later_records(
+            http_mocker,
+            string_to_datetime(self._config["start_date"]),
+            pendulum.duration(weeks=12),
+            api_token_authenticator,
+        )
+
+        state_value = {
+            "updated_at": datetime_to_string(pendulum.now(tz="UTC").subtract(years=1, weeks=50))
+        }
+
+        state = StateBuilder().with_stream_state("groups", state_value).build()
+
+        output = read_stream("groups", SyncMode.full_refresh, self._config, state=state)
+        assert len(output.records) == 1

airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/utils.py

@@ -4,7 +4,7 @@
 from typing import Any, Dict, List, Optional
 
 import pendulum
-from airbyte_cdk.models import AirbyteMessage
+from airbyte_cdk.models import AirbyteMessage, AirbyteStateMessage
 from airbyte_cdk.models import Level as LogLevel
 from airbyte_cdk.models import SyncMode
 from airbyte_cdk.test.catalog_builder import CatalogBuilder
@@ -14,7 +14,11 @@
 
 
 def read_stream(
-    stream_name: str, sync_mode: SyncMode, config: Dict[str, Any], state: Optional[Dict[str, Any]] = None, expecting_exception: bool = False
+    stream_name: str,
+    sync_mode: SyncMode,
+    config: Dict[str, Any],
+    state: Optional[List[AirbyteStateMessage]] = None,
+    expecting_exception: bool = False,
 ) -> EntrypointOutput:
     catalog = CatalogBuilder().with_stream(stream_name, sync_mode).build()
     return read(SourceZendeskSupport(config=config, catalog=catalog, state=state), config, catalog, state, expecting_exception)

airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/zs_requests/__init__.py

@@ -1,3 +1,4 @@
+from .groups_request_builder import GroupsRequestBuilder
 from .post_comment_votes_request_builder import PostCommentVotesRequestBuilder
 from .post_comments_request_builder import PostsCommentsRequestBuilder
 from .post_votes_request_builder import PostsVotesRequestBuilder

airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/zs_requests/groups_request_builder.py

@@ -0,0 +1,29 @@
+# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
+
+import calendar
+
+import pendulum
+
+from .base_request_builder import ZendeskSupportBaseRequestBuilder
+from .request_authenticators.authenticator import Authenticator
+
+
+class GroupsRequestBuilder(ZendeskSupportBaseRequestBuilder):
+    @classmethod
+    def groups_endpoint(cls, authenticator: Authenticator) -> "GroupsRequestBuilder":
+        return cls("d3v-airbyte", "groups").with_authenticator(authenticator)
+
+    def __init__(self, subdomain: str, resource: str) -> None:
+        super().__init__(subdomain, resource)
+        self._page_size: int = None
+
+    @property
+    def query_params(self):
+        params = super().query_params or {}
+        if self._page_size:
+            params["per_page"] = self._page_size
+        return params
+
+    def with_page_size(self, page_size: int) -> "PostCommentVotesRequestBuilder":
+        self._page_size: int = page_size
+        return self

airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/zs_responses/__init__.py

@@ -1,4 +1,5 @@
 from .error_response_builder import ErrorResponseBuilder
+from .groups_response_builder import GroupsResponseBuilder
 from .post_comment_votes_response_builder import PostCommentVotesResponseBuilder
 from .post_comments_response_builder import PostsCommentsResponseBuilder
 from .post_votes_response_builder import PostsVotesResponseBuilder

airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/zs_responses/groups_response_builder.py

@@ -0,0 +1,11 @@
+# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
+
+from airbyte_cdk.test.mock_http.response_builder import FieldPath, HttpResponseBuilder, find_template
+
+from .pagination_strategies import CursorBasedPaginationStrategy
+
+
+class GroupsResponseBuilder(HttpResponseBuilder):
+    @classmethod
+    def groups_response(cls) -> "GroupsResponseBuilder":
+        return cls(find_template("groups", __file__), FieldPath("groups"), CursorBasedPaginationStrategy())

airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/zs_responses/records/__init__.py

@@ -1,3 +1,4 @@
+from .groups_records_builder import GroupsRecordBuilder
 from .post_comment_votes_records_builder import PostCommentVotesRecordBuilder
 from .post_comments_records_builder import PostsCommentsRecordBuilder
 from .post_votes_records_builder import PostsVotesRecordBuilder

airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/zs_responses/records/groups_records_builder.py

@@ -0,0 +1,12 @@
+# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
+
+from airbyte_cdk.test.mock_http.response_builder import FieldPath, NestedPath
+
+from .records_builder import ZendeskSupportRecordBuilder
+
+
+class GroupsRecordBuilder(ZendeskSupportRecordBuilder):
+    @classmethod
+    def groups_record(cls) -> "GroupsRecordBuilder":
+        record_template = cls.extract_record("groups", __file__, NestedPath(["groups", 0]))
+        return cls(record_template, FieldPath("id"), FieldPath("updated_at"))

airbyte-integrations/connectors/source-zendesk-support/unit_tests/resource/http/response/groups.json

@@ -0,0 +1,15 @@
+{
+  "groups": [
+    {
+      "created_at": "2024-11-20T00:00:00Z",
+      "default": true,
+      "deleted": false,
+      "description": "Employed",
+      "id": 3432,
+      "is_public": true,
+      "name": "Remote Employees",
+      "updated_at": "2024-11-20T00:00:00Z",
+      "url": "https://company.zendesk.com/api/v2/groups/3432.json"
+    }
+  ]
+}