diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 276afe14ffcc..5a7b94dacbf0 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -785,7 +785,7 @@ - name: Jira sourceDefinitionId: 68e63de2-bb83-4c7e-93fa-a8a9051e3993 dockerRepository: airbyte/source-jira - dockerImageTag: 0.3.1 + dockerImageTag: 0.3.2 documentationUrl: https://docs.airbyte.com/integrations/sources/jira icon: jira.svg sourceType: api diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index b9c3688da03f..2b40465fb7c3 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -6677,7 +6677,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-jira:0.3.1" +- dockerImage: "airbyte/source-jira:0.3.2" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/jira" connectionSpecification: diff --git a/airbyte-integrations/connectors/source-jira/Dockerfile b/airbyte-integrations/connectors/source-jira/Dockerfile index b74fa3edf0c4..0df724eff5a0 100644 --- a/airbyte-integrations/connectors/source-jira/Dockerfile +++ b/airbyte-integrations/connectors/source-jira/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.3.1 +LABEL io.airbyte.version=0.3.2 LABEL io.airbyte.name=airbyte/source-jira diff --git a/airbyte-integrations/connectors/source-jira/source_jira/streams.py b/airbyte-integrations/connectors/source-jira/source_jira/streams.py index 45fde4ece011..ac8e08330cab 100644 --- a/airbyte-integrations/connectors/source-jira/source_jira/streams.py +++ b/airbyte-integrations/connectors/source-jira/source_jira/streams.py @@ -43,13 +43,16 @@ def url_base(self) -> str: def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: response_json = response.json() if isinstance(response_json, dict): - if response_json.get("isLast"): - return startAt = response_json.get("startAt") if startAt is not None: startAt += response_json["maxResults"] - if startAt < response_json["total"]: - return {"startAt": startAt} + if "isLast" in response_json: + if response_json["isLast"]: + return + elif "total" in response_json: + if startAt >= response_json["total"]: + return + return {"startAt": startAt} elif isinstance(response_json, list): if len(response_json) == self.page_size: query_params = dict(parse_qsl(urlparse.urlparse(response.url).query)) @@ -553,6 +556,9 @@ def read_records(self, stream_slice: Optional[Mapping[str, Any]] = None, **kwarg for issue in read_full_refresh(self.issues_stream): yield from super().read_records(stream_slice={"key": issue["key"]}, **kwargs) + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + return None + class IssueResolutions(JiraStream): """ diff --git a/airbyte-integrations/connectors/source-jira/unit_tests/test_pagination.py b/airbyte-integrations/connectors/source-jira/unit_tests/test_pagination.py new file mode 100644 index 000000000000..a370342663d2 --- /dev/null +++ b/airbyte-integrations/connectors/source-jira/unit_tests/test_pagination.py @@ -0,0 +1,89 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import json +from http import HTTPStatus + +import responses +from source_jira.streams import Issues, Projects, Users +from source_jira.utils import read_full_refresh + + +@responses.activate +def test_pagination_projects(): + domain = "domain.com" + responses_json = [ + (HTTPStatus.OK, {}, json.dumps({"startAt": 0, "maxResults": 2, "total": 6, "isLast": False, "values": [{"id": "1"}, {"id": "2"}]})), + (HTTPStatus.OK, {}, json.dumps({"startAt": 2, "maxResults": 2, "total": 6, "isLast": False, "values": [{"id": "3"}, {"id": "4"}]})), + (HTTPStatus.OK, {}, json.dumps({"startAt": 4, "maxResults": 2, "total": 6, "isLast": True, "values": [{"id": "5"}, {"id": "6"}]})), + ] + + responses.add_callback( + responses.GET, + f"https://{domain}/rest/api/3/project/search", + callback=lambda request: responses_json.pop(0), + content_type="application/json", + ) + + stream = Projects(authenticator=None, domain=domain, projects=[]) + records = list(read_full_refresh(stream)) + assert records == [{"id": "1"}, {"id": "2"}, {"id": "3"}, {"id": "4"}, {"id": "5"}, {"id": "6"}] + + +@responses.activate +def test_pagination_issues(): + domain = "domain.com" + responses_json = [ + (HTTPStatus.OK, {}, json.dumps({"startAt": 0, "maxResults": 2, "total": 6, "issues": [{"id": "1", "updated": "2022-01-01"}, {"id": "2", "updated": "2022-01-01"}]})), + (HTTPStatus.OK, {}, json.dumps({"startAt": 2, "maxResults": 2, "total": 6, "issues": [{"id": "3", "updated": "2022-01-01"}, {"id": "4", "updated": "2022-01-01"}]})), + (HTTPStatus.OK, {}, json.dumps({"startAt": 4, "maxResults": 2, "total": 6, "issues": [{"id": "5", "updated": "2022-01-01"}, {"id": "6", "updated": "2022-01-01"}]})), + ] + + responses.add_callback( + responses.GET, + f"https://{domain}/rest/api/3/search", + callback=lambda request: responses_json.pop(0), + content_type="application/json", + ) + + stream = Issues(authenticator=None, domain=domain, projects=[]) + stream.transform = lambda record, **kwargs: record + records = list(read_full_refresh(stream)) + assert records == [ + {"id": "1", "updated": "2022-01-01"}, + {"id": "2", "updated": "2022-01-01"}, + {"id": "3", "updated": "2022-01-01"}, + {"id": "4", "updated": "2022-01-01"}, + {"id": "5", "updated": "2022-01-01"}, + {"id": "6", "updated": "2022-01-01"} + ] + + +@responses.activate +def test_pagination_users(): + domain = "domain.com" + responses_json = [ + (HTTPStatus.OK, {}, json.dumps([{"self": "user1"}, {"self": "user2"}])), + (HTTPStatus.OK, {}, json.dumps([{"self": "user3"}, {"self": "user4"}])), + (HTTPStatus.OK, {}, json.dumps([{"self": "user5"}])), + ] + + responses.add_callback( + responses.GET, + f"https://{domain}/rest/api/3/users/search", + callback=lambda request: responses_json.pop(0), + content_type="application/json", + ) + + stream = Users(authenticator=None, domain=domain, projects=[]) + stream.page_size = 2 + records = list(read_full_refresh(stream)) + assert records == [ + {"self": "user1"}, + {"self": "user2"}, + {"self": "user3"}, + {"self": "user4"}, + {"self": "user5"}, + ] diff --git a/docs/integrations/sources/jira.md b/docs/integrations/sources/jira.md index 423d4cef658c..046f002ac134 100644 --- a/docs/integrations/sources/jira.md +++ b/docs/integrations/sources/jira.md @@ -126,6 +126,7 @@ The Jira connector should not run into Jira API limitations under normal usage. | Version | Date | Pull Request | Subject | |:--------|:-----------|:------------------------------------------------------------|:------------------------------------------------------------------------------------------------------------------------| +| 0.3.2 | 2022-12-23 | [\#20859](https://github.com/airbytehq/airbyte/pull/20859) | Fixed pagination for streams `issue_remote_links`, `sprints` | | 0.3.1 | 2022-12-14 | [\#20128](https://github.com/airbytehq/airbyte/pull/20128) | Improved code to become beta | | 0.3.0 | 2022-11-03 | [\#18901](https://github.com/airbytehq/airbyte/pull/18901) | Adds UserGroupsDetailed schema, fix Incremental normalization, add Incremental support for IssueComments, IssueWorklogs | | 0.2.23 | 2022-10-28 | [\#18505](https://github.com/airbytehq/airbyte/pull/18505) | Correcting `max_results` bug introduced in connector stream |