Source Jira: Fixed pagination for streams issue_remote_links, sprints

airbyte-config/init/src/main/resources/seed/source_definitions.yaml

@@ -785,7 +785,7 @@
 - name: Jira
   sourceDefinitionId: 68e63de2-bb83-4c7e-93fa-a8a9051e3993
   dockerRepository: airbyte/source-jira
-  dockerImageTag: 0.3.1
+  dockerImageTag: 0.3.2
   documentationUrl: https://docs.airbyte.com/integrations/sources/jira
   icon: jira.svg
   sourceType: api

airbyte-config/init/src/main/resources/seed/source_specs.yaml

@@ -6677,7 +6677,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-jira:0.3.1"
+- dockerImage: "airbyte/source-jira:0.3.2"
   spec:
     documentationUrl: "https://docs.airbyte.com/integrations/sources/jira"
     connectionSpecification:

airbyte-integrations/connectors/source-jira/Dockerfile

@@ -12,5 +12,5 @@ RUN pip install .
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.3.1
+LABEL io.airbyte.version=0.3.2
 LABEL io.airbyte.name=airbyte/source-jira

airbyte-integrations/connectors/source-jira/source_jira/streams.py

@@ -43,13 +43,16 @@ def url_base(self) -> str:
     def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]:
         response_json = response.json()
         if isinstance(response_json, dict):
-            if response_json.get("isLast"):
-                return
             startAt = response_json.get("startAt")
             if startAt is not None:
                 startAt += response_json["maxResults"]
-                if startAt < response_json["total"]:
-                    return {"startAt": startAt}
+                if "isLast" in response_json:
+                    if response_json["isLast"]:
+                        return
+                elif "total" in response_json:
+                    if startAt >= response_json["total"]:
+                        return
+                return {"startAt": startAt}
         elif isinstance(response_json, list):
             if len(response_json) == self.page_size:
                 query_params = dict(parse_qsl(urlparse.urlparse(response.url).query))
@@ -553,6 +556,9 @@ def read_records(self, stream_slice: Optional[Mapping[str, Any]] = None, **kwarg
         for issue in read_full_refresh(self.issues_stream):
             yield from super().read_records(stream_slice={"key": issue["key"]}, **kwargs)
 
+    def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]:
+        return None
+
 
 class IssueResolutions(JiraStream):
     """

airbyte-integrations/connectors/source-jira/unit_tests/test_pagination.py

@@ -0,0 +1,89 @@
+#
+# Copyright (c) 2022 Airbyte, Inc., all rights reserved.
+#
+
+
+import json
+from http import HTTPStatus
+
+import responses
+from source_jira.streams import Issues, Projects, Users
+from source_jira.utils import read_full_refresh
+
+
+@responses.activate
+def test_pagination_projects():
+    domain = "domain.com"
+    responses_json = [
+        (HTTPStatus.OK, {}, json.dumps({"startAt": 0, "maxResults": 2, "total": 6, "isLast": False, "values": [{"id": "1"}, {"id": "2"}]})),
+        (HTTPStatus.OK, {}, json.dumps({"startAt": 2, "maxResults": 2, "total": 6, "isLast": False, "values": [{"id": "3"}, {"id": "4"}]})),
+        (HTTPStatus.OK, {}, json.dumps({"startAt": 4, "maxResults": 2, "total": 6, "isLast": True, "values": [{"id": "5"}, {"id": "6"}]})),
+    ]
+
+    responses.add_callback(
+        responses.GET,
+        f"https://{domain}/rest/api/3/project/search",
+        callback=lambda request: responses_json.pop(0),
+        content_type="application/json",
+    )
+
+    stream = Projects(authenticator=None, domain=domain, projects=[])
+    records = list(read_full_refresh(stream))
+    assert records == [{"id": "1"}, {"id": "2"}, {"id": "3"}, {"id": "4"}, {"id": "5"}, {"id": "6"}]
+
+
+@responses.activate
+def test_pagination_issues():
+    domain = "domain.com"
+    responses_json = [
+        (HTTPStatus.OK, {}, json.dumps({"startAt": 0, "maxResults": 2, "total": 6, "issues": [{"id": "1", "updated": "2022-01-01"}, {"id": "2", "updated": "2022-01-01"}]})),
+        (HTTPStatus.OK, {}, json.dumps({"startAt": 2, "maxResults": 2, "total": 6, "issues": [{"id": "3", "updated": "2022-01-01"}, {"id": "4", "updated": "2022-01-01"}]})),
+        (HTTPStatus.OK, {}, json.dumps({"startAt": 4, "maxResults": 2, "total": 6, "issues": [{"id": "5", "updated": "2022-01-01"}, {"id": "6", "updated": "2022-01-01"}]})),
+    ]
+
+    responses.add_callback(
+        responses.GET,
+        f"https://{domain}/rest/api/3/search",
+        callback=lambda request: responses_json.pop(0),
+        content_type="application/json",
+    )
+
+    stream = Issues(authenticator=None, domain=domain, projects=[])
+    stream.transform = lambda record, **kwargs: record
+    records = list(read_full_refresh(stream))
+    assert records == [
+        {"id": "1", "updated": "2022-01-01"},
+        {"id": "2", "updated": "2022-01-01"},
+        {"id": "3", "updated": "2022-01-01"},
+        {"id": "4", "updated": "2022-01-01"},
+        {"id": "5", "updated": "2022-01-01"},
+        {"id": "6", "updated": "2022-01-01"}
+    ]
+
+
+@responses.activate
+def test_pagination_users():
+    domain = "domain.com"
+    responses_json = [
+        (HTTPStatus.OK, {}, json.dumps([{"self": "user1"}, {"self": "user2"}])),
+        (HTTPStatus.OK, {}, json.dumps([{"self": "user3"}, {"self": "user4"}])),
+        (HTTPStatus.OK, {}, json.dumps([{"self": "user5"}])),
+    ]
+
+    responses.add_callback(
+        responses.GET,
+        f"https://{domain}/rest/api/3/users/search",
+        callback=lambda request: responses_json.pop(0),
+        content_type="application/json",
+    )
+
+    stream = Users(authenticator=None, domain=domain, projects=[])
+    stream.page_size = 2
+    records = list(read_full_refresh(stream))
+    assert records == [
+        {"self": "user1"},
+        {"self": "user2"},
+        {"self": "user3"},
+        {"self": "user4"},
+        {"self": "user5"},
+    ]

docs/integrations/sources/jira.md

@@ -126,6 +126,7 @@ The Jira connector should not run into Jira API limitations under normal usage.
 
 | Version | Date       | Pull Request                                                | Subject                                                                                                                 |
 |:--------|:-----------|:------------------------------------------------------------|:------------------------------------------------------------------------------------------------------------------------|
+| 0.3.2   | 2022-12-23 | [\#20859](https://github.com/airbytehq/airbyte/pull/20859)  | Fixed pagination for streams `issue_remote_links`, `sprints`                                                            |
 | 0.3.1   | 2022-12-14 | [\#20128](https://github.com/airbytehq/airbyte/pull/20128)  | Improved code to become beta                                                                                            |
 | 0.3.0   | 2022-11-03 | [\#18901](https://github.com/airbytehq/airbyte/pull/18901)  | Adds UserGroupsDetailed schema, fix Incremental normalization, add Incremental support for IssueComments, IssueWorklogs |
 | 0.2.23  | 2022-10-28 | [\#18505](https://github.com/airbytehq/airbyte/pull/18505)  | Correcting `max_results` bug introduced in connector stream                                                             |