From 3c5a98141ed0193dfd1f1ece42d0914d026e20ba Mon Sep 17 00:00:00 2001 From: Chris Wu Date: Tue, 28 Sep 2021 16:29:56 -0700 Subject: [PATCH 1/7] Add caching for Jira source and add Boards stream --- .../integration_tests/configured_catalog.json | 10 ++ .../source_jira/schemas/boards.json | 18 ++++ .../source-jira/source_jira/source.py | 2 + .../source-jira/source_jira/streams.py | 98 ++++++++++++++++++- 4 files changed, 126 insertions(+), 2 deletions(-) create mode 100644 airbyte-integrations/connectors/source-jira/source_jira/schemas/boards.json diff --git a/airbyte-integrations/connectors/source-jira/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-jira/integration_tests/configured_catalog.json index afbacd25e56bb..d81d924cd10d0 100644 --- a/airbyte-integrations/connectors/source-jira/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-jira/integration_tests/configured_catalog.json @@ -20,6 +20,16 @@ "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" }, + { + "stream": { + "name": "boards", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_cursor": false + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, { "stream": { "name": "dashboards", diff --git a/airbyte-integrations/connectors/source-jira/source_jira/schemas/boards.json b/airbyte-integrations/connectors/source-jira/source_jira/schemas/boards.json new file mode 100644 index 0000000000000..0ddc9a329cbaa --- /dev/null +++ b/airbyte-integrations/connectors/source-jira/source_jira/schemas/boards.json @@ -0,0 +1,18 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "properties": { + "id": { + "type": "integer" + }, + "self": { + "type": "string" + }, + "name": { + "type": "string" + }, + "type": { + "type": "string" + } + } +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-jira/source_jira/source.py b/airbyte-integrations/connectors/source-jira/source_jira/source.py index e6d1e47113363..57948c160082f 100644 --- a/airbyte-integrations/connectors/source-jira/source_jira/source.py +++ b/airbyte-integrations/connectors/source-jira/source_jira/source.py @@ -35,6 +35,7 @@ from .streams import ( ApplicationRoles, Avatars, + Boards, Dashboards, Filters, FilterSharing, @@ -118,6 +119,7 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: return [ ApplicationRoles(**args), Avatars(**args), + Boards(**args), Dashboards(**args), Filters(**args), FilterSharing(**args), diff --git a/airbyte-integrations/connectors/source-jira/source_jira/streams.py b/airbyte-integrations/connectors/source-jira/source_jira/streams.py index b61342017cbe8..5f06602cabf43 100644 --- a/airbyte-integrations/connectors/source-jira/source_jira/streams.py +++ b/airbyte-integrations/connectors/source-jira/source_jira/streams.py @@ -22,6 +22,7 @@ # SOFTWARE. # +import os import urllib.parse as urlparse from abc import ABC, abstractmethod from typing import Any, Iterable, Mapping, MutableMapping, Optional @@ -29,20 +30,48 @@ import pendulum import requests +import vcr from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.streams.http import HttpStream +from vcr.cassette import Cassette API_VERSION = 3 +def request_cache() -> Cassette: + """ + Builds VCR instance. + It deletes file everytime we create it, normally should be called only once. + We can't use NamedTemporaryFile here because yaml serializer doesn't work well with empty files. + """ + filename = "request_cache.yml" + try: + os.remove(filename) + except FileNotFoundError: + pass + + return vcr.use_cassette(str(filename), record_mode="new_episodes", serializer="yaml") + + class JiraStream(HttpStream, ABC): """ Jira API Reference: https://developer.atlassian.com/cloud/jira/platform/rest/v3/intro/ """ + cache = request_cache() primary_key = "id" parse_response_root = None + # To prevent dangerous behavior, the `vcr` library prohibits the use of nested caching. + # Here's an example of dangerous behavior: + # cache = Cassette.use('whatever') + # with cache: + # with cache: + # pass + # + # Therefore, we will only use `cache` for the top-level stream, so as not to cause possible difficulties. + top_level_stream = True + def __init__(self, domain: str, **kwargs): super(JiraStream, self).__init__(**kwargs) self._domain = domain @@ -86,13 +115,29 @@ def request_headers( ) -> Mapping[str, Any]: return {"Accept": "application/json"} + def read_records(self, **kwargs) -> Iterable[Mapping[str, Any]]: + if self.top_level_stream: + with self.cache: + yield from super().read_records(**kwargs) + else: + yield from super().read_records(**kwargs) + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: response_json = response.json() records = response_json if not self.parse_response_root else response_json.get(self.parse_response_root, []) if isinstance(records, list): - yield from records + for record in records: + yield self.transform(record=record, **kwargs) else: - yield records + yield self.transform(record=records, **kwargs) + + def transform(self, record: MutableMapping[str, Any], **kwargs) -> MutableMapping[str, Any]: + return record + +class V1ApiJiraStream(JiraStream, ABC): + @property + def url_base(self) -> str: + return f"https://{self._domain}/rest/agile/1.0/" class IncrementalJiraStream(JiraStream, ABC): @@ -140,6 +185,32 @@ def read_records(self, stream_slice: Optional[Mapping[str, Any]] = None, **kwarg yield from super().read_records(stream_slice={"avatar_type": avatar_type}, **kwargs) +class Boards(V1ApiJiraStream): + """ + https://developer.atlassian.com/cloud/jira/software/rest/api-group-other-operations/#api-agile-1-0-board-get + """ + + parse_response_root = "values" + top_level_stream = False + + def path(self, **kwargs) -> str: + return "board" + + def request_params(self, stream_slice: Mapping[str, Any], **kwargs): + params = super().request_params(stream_slice=stream_slice, **kwargs) + params["projectKeyOrId"] = stream_slice["project_id"] + return params + + def read_records(self, stream_slice: Optional[Mapping[str, Any]] = None, **kwargs) -> Iterable[Mapping[str, Any]]: + projects_stream = Projects(authenticator=self.authenticator, domain=self._domain) + for project in projects_stream.read_records(sync_mode=SyncMode.full_refresh): + yield from super().read_records(stream_slice={"project_id": project["id"]}, **kwargs) + + def transform(self, record: MutableMapping[str, Any], stream_slice: Mapping[str, Any], **kwargs) -> MutableMapping[str, Any]: + record["project_id"] = stream_slice["project_id"] + return record + + class Dashboards(JiraStream): """ https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-dashboards/#api-rest-api-3-dashboard-get @@ -167,6 +238,8 @@ class FilterSharing(JiraStream): https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-filter-sharing/#api-rest-api-3-filter-id-permission-get """ + top_level_stream = False + def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: filter_id = stream_slice["filter_id"] return f"filter/{filter_id}/permission" @@ -225,6 +298,7 @@ class IssueComments(JiraStream): """ parse_response_root = "comments" + top_level_stream = False def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: key = stream_slice["key"] @@ -262,6 +336,7 @@ class IssueCustomFieldContexts(JiraStream): """ parse_response_root = "values" + top_level_stream = False def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: field_id = stream_slice["field_id"] @@ -335,6 +410,8 @@ class IssueProperties(JiraStream): https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-issue-properties/#api-rest-api-3-issue-issueidorkey-properties-propertykey-get """ + top_level_stream = False + def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: key = stream_slice["key"] issue_key = stream_slice["issue_key"] @@ -353,6 +430,8 @@ class IssueRemoteLinks(JiraStream): https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-issue-remote-links/#api-rest-api-3-issue-issueidorkey-remotelink-get """ + top_level_stream = False + def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: key = stream_slice["key"] return f"issue/{key}/remotelink" @@ -416,6 +495,7 @@ class IssueVotes(JiraStream): """ # parse_response_root = "voters" + top_level_stream = False def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: key = stream_slice["key"] @@ -435,6 +515,7 @@ class IssueWatchers(JiraStream): """ # parse_response_root = "watchers" + top_level_stream = False def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: key = stream_slice["key"] @@ -452,6 +533,7 @@ class IssueWorklogs(JiraStream): """ parse_response_root = "worklogs" + top_level_stream = False def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: key = stream_slice["key"] @@ -524,6 +606,8 @@ class ProjectAvatars(JiraStream): https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-project-avatars/#api-rest-api-3-project-projectidorkey-avatars-get """ + top_level_stream = False + def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: key = stream_slice["key"] return f"project/{key}/avatars" @@ -554,6 +638,7 @@ class ProjectComponents(JiraStream): """ parse_response_root = "values" + top_level_stream = False def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: key = stream_slice["key"] @@ -570,6 +655,8 @@ class ProjectEmail(JiraStream): https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-project-email/#api-rest-api-3-project-projectid-email-get """ + top_level_stream = False + def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: project_id = stream_slice["project_id"] return f"project/{project_id}/email" @@ -585,6 +672,8 @@ class ProjectPermissionSchemes(JiraStream): https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-project-permission-schemes/#api-rest-api-3-project-projectkeyorid-securitylevel-get """ + top_level_stream = False + def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: key = stream_slice["key"] return f"project/{key}/securitylevel" @@ -610,6 +699,7 @@ class ProjectVersions(JiraStream): """ parse_response_root = "values" + top_level_stream = False def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: key = stream_slice["key"] @@ -637,6 +727,8 @@ class ScreenTabs(JiraStream): https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-screen-tabs/#api-rest-api-3-screens-screenid-tabs-get """ + top_level_stream = False + def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: screen_id = stream_slice["screen_id"] return f"screens/{screen_id}/tabs" @@ -656,6 +748,8 @@ class ScreenTabFields(JiraStream): https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-screen-tab-fields/#api-rest-api-3-screens-screenid-tabs-tabid-fields-get """ + top_level_stream = False + def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: screen_id = stream_slice["screen_id"] tab_id = stream_slice["tab_id"] From c7a17abbf29b45c56e46c29a756f4adc0c1986b7 Mon Sep 17 00:00:00 2001 From: Chris Wu Date: Tue, 28 Sep 2021 16:46:25 -0700 Subject: [PATCH 2/7] Fix draft version --- .../connectors/source-jira/source_jira/schemas/boards.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/source-jira/source_jira/schemas/boards.json b/airbyte-integrations/connectors/source-jira/source_jira/schemas/boards.json index 0ddc9a329cbaa..6c1f0454d7b95 100644 --- a/airbyte-integrations/connectors/source-jira/source_jira/schemas/boards.json +++ b/airbyte-integrations/connectors/source-jira/source_jira/schemas/boards.json @@ -1,5 +1,5 @@ { - "$schema": "http://json-schema.org/draft-04/schema#", + "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "properties": { "id": { @@ -15,4 +15,4 @@ "type": "string" } } -} \ No newline at end of file +} From d6baff3437ced1611b4ee76c4976778442eb4f28 Mon Sep 17 00:00:00 2001 From: Chris Wu Date: Tue, 28 Sep 2021 17:44:22 -0700 Subject: [PATCH 3/7] Add sprints stream --- .../integration_tests/configured_catalog.json | 10 ++++++ .../source_jira/schemas/boards.json | 3 ++ .../source_jira/schemas/sprints.json | 36 +++++++++++++++++++ .../source-jira/source_jira/source.py | 2 ++ .../source-jira/source_jira/streams.py | 20 ++++++++++- 5 files changed, 70 insertions(+), 1 deletion(-) create mode 100644 airbyte-integrations/connectors/source-jira/source_jira/schemas/sprints.json diff --git a/airbyte-integrations/connectors/source-jira/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-jira/integration_tests/configured_catalog.json index d81d924cd10d0..7e6c0d92d9de7 100644 --- a/airbyte-integrations/connectors/source-jira/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-jira/integration_tests/configured_catalog.json @@ -150,6 +150,16 @@ "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" }, + { + "stream": { + "name": "sprints", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_cursor": false + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, { "stream": { "name": "time_tracking", diff --git a/airbyte-integrations/connectors/source-jira/source_jira/schemas/boards.json b/airbyte-integrations/connectors/source-jira/source_jira/schemas/boards.json index 6c1f0454d7b95..c7f71ad2e25d0 100644 --- a/airbyte-integrations/connectors/source-jira/source_jira/schemas/boards.json +++ b/airbyte-integrations/connectors/source-jira/source_jira/schemas/boards.json @@ -13,6 +13,9 @@ }, "type": { "type": "string" + }, + "projectId": { + "type": "string" } } } diff --git a/airbyte-integrations/connectors/source-jira/source_jira/schemas/sprints.json b/airbyte-integrations/connectors/source-jira/source_jira/schemas/sprints.json new file mode 100644 index 0000000000000..7ee89b85aec20 --- /dev/null +++ b/airbyte-integrations/connectors/source-jira/source_jira/schemas/sprints.json @@ -0,0 +1,36 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": "integer" + }, + "self": { + "type": "string" + }, + "state": { + "type": "string" + }, + "name": { + "type": "string" + }, + "startDate": { + "type": "string", + "format": "date-time" + }, + "endDate": { + "type": "string", + "format": "date-time" + }, + "completeDate": { + "type": "string", + "format": "date-time" + }, + "originBoardId": { + "type": "integer" + }, + "goal": { + "type": "string" + } + } +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-jira/source_jira/source.py b/airbyte-integrations/connectors/source-jira/source_jira/source.py index 57948c160082f..a2916008862af 100644 --- a/airbyte-integrations/connectors/source-jira/source_jira/source.py +++ b/airbyte-integrations/connectors/source-jira/source_jira/source.py @@ -74,6 +74,7 @@ ScreenSchemes, ScreenTabFields, ScreenTabs, + Sprints, TimeTracking, Users, Workflows, @@ -158,6 +159,7 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: ScreenTabs(**args), ScreenTabFields(**args), ScreenSchemes(**args), + Sprints(**args), TimeTracking(**args), Users(**args), Workflows(**args), diff --git a/airbyte-integrations/connectors/source-jira/source_jira/streams.py b/airbyte-integrations/connectors/source-jira/source_jira/streams.py index 5f06602cabf43..b3d77920b1267 100644 --- a/airbyte-integrations/connectors/source-jira/source_jira/streams.py +++ b/airbyte-integrations/connectors/source-jira/source_jira/streams.py @@ -207,7 +207,7 @@ def read_records(self, stream_slice: Optional[Mapping[str, Any]] = None, **kwarg yield from super().read_records(stream_slice={"project_id": project["id"]}, **kwargs) def transform(self, record: MutableMapping[str, Any], stream_slice: Mapping[str, Any], **kwargs) -> MutableMapping[str, Any]: - record["project_id"] = stream_slice["project_id"] + record["projectId"] = stream_slice["project_id"] return record @@ -773,6 +773,24 @@ class ScreenSchemes(JiraStream): def path(self, **kwargs) -> str: return "screenscheme" +class Sprints(V1ApiJiraStream): + """ + https://developer.atlassian.com/cloud/jira/software/rest/api-group-board/#api-agile-1-0-board-boardid-sprint-get + """ + + parse_response_root = "values" + raise_on_http_errors = False + top_level_stream = False + + def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: + board_id = stream_slice["board_id"] + return f"board/{board_id}/sprint" + + def read_records(self, stream_slice: Optional[Mapping[str, Any]] = None, **kwargs) -> Iterable[Mapping[str, Any]]: + boards_stream = Boards(authenticator=self.authenticator, domain=self._domain) + for board in boards_stream.read_records(sync_mode=SyncMode.full_refresh): + yield from super().read_records(stream_slice={"board_id": board["id"]}, **kwargs) + class TimeTracking(JiraStream): """ From 16313d818206bfdaa391e9db67b5e7643f6c6061 Mon Sep 17 00:00:00 2001 From: Chris Wu Date: Thu, 30 Sep 2021 10:37:25 -0700 Subject: [PATCH 4/7] Add vcr to setup.py --- airbyte-integrations/connectors/source-jira/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-jira/setup.py b/airbyte-integrations/connectors/source-jira/setup.py index ffdd75a87b5c3..cb65ff061c845 100644 --- a/airbyte-integrations/connectors/source-jira/setup.py +++ b/airbyte-integrations/connectors/source-jira/setup.py @@ -25,7 +25,7 @@ from setuptools import find_packages, setup -MAIN_REQUIREMENTS = ["airbyte-cdk~=0.1", "requests==2.25.1", "pendulum>=1.2.0"] +MAIN_REQUIREMENTS = ["airbyte-cdk~=0.1", "requests==2.25.1", "pendulum>=1.2.0", "vcrpy==4.1.1"] TEST_REQUIREMENTS = [ "pytest==6.1.2", From 158ed6abec46f7d3c9e787012171a378c52eaefa Mon Sep 17 00:00:00 2001 From: Chris Wu Date: Thu, 30 Sep 2021 11:03:30 -0700 Subject: [PATCH 5/7] Formatting --- airbyte-cdk/python/airbyte_cdk/sources/utils/transform.py | 1 + airbyte-cdk/python/unit_tests/sources/utils/test_transform.py | 1 + .../connectors/source-jira/source_jira/schemas/sprints.json | 2 +- .../connectors/source-jira/source_jira/streams.py | 2 ++ .../unit_tests/test_incremental_streams.py | 3 +-- 5 files changed, 6 insertions(+), 3 deletions(-) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/utils/transform.py b/airbyte-cdk/python/airbyte_cdk/sources/utils/transform.py index 607b5b680f147..60cbb47e22bd2 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/utils/transform.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/utils/transform.py @@ -21,6 +21,7 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. # + from distutils.util import strtobool from enum import Flag, auto from typing import Any, Callable, Dict diff --git a/airbyte-cdk/python/unit_tests/sources/utils/test_transform.py b/airbyte-cdk/python/unit_tests/sources/utils/test_transform.py index d2d7b70668b85..473144b31ce22 100644 --- a/airbyte-cdk/python/unit_tests/sources/utils/test_transform.py +++ b/airbyte-cdk/python/unit_tests/sources/utils/test_transform.py @@ -21,6 +21,7 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. # + import json import pytest diff --git a/airbyte-integrations/connectors/source-jira/source_jira/schemas/sprints.json b/airbyte-integrations/connectors/source-jira/source_jira/schemas/sprints.json index 7ee89b85aec20..d91f1e1ff40f7 100644 --- a/airbyte-integrations/connectors/source-jira/source_jira/schemas/sprints.json +++ b/airbyte-integrations/connectors/source-jira/source_jira/schemas/sprints.json @@ -33,4 +33,4 @@ "type": "string" } } -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-jira/source_jira/streams.py b/airbyte-integrations/connectors/source-jira/source_jira/streams.py index b3d77920b1267..b65ef19fd55f8 100644 --- a/airbyte-integrations/connectors/source-jira/source_jira/streams.py +++ b/airbyte-integrations/connectors/source-jira/source_jira/streams.py @@ -134,6 +134,7 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp def transform(self, record: MutableMapping[str, Any], **kwargs) -> MutableMapping[str, Any]: return record + class V1ApiJiraStream(JiraStream, ABC): @property def url_base(self) -> str: @@ -773,6 +774,7 @@ class ScreenSchemes(JiraStream): def path(self, **kwargs) -> str: return "screenscheme" + class Sprints(V1ApiJiraStream): """ https://developer.atlassian.com/cloud/jira/software/rest/api-group-board/#api-agile-1-0-board-boardid-sprint-get diff --git a/airbyte-integrations/connectors/source-scaffold-source-http/unit_tests/test_incremental_streams.py b/airbyte-integrations/connectors/source-scaffold-source-http/unit_tests/test_incremental_streams.py index 5bb2c4081696e..f0b1df433532a 100644 --- a/airbyte-integrations/connectors/source-scaffold-source-http/unit_tests/test_incremental_streams.py +++ b/airbyte-integrations/connectors/source-scaffold-source-http/unit_tests/test_incremental_streams.py @@ -22,9 +22,8 @@ # SOFTWARE. # -from pytest import fixture - from airbyte_cdk.models import SyncMode +from pytest import fixture from source_scaffold_source_http.source import IncrementalScaffoldSourceHttpStream From b8817a8d61822e9b236861d333f6c396e016b026 Mon Sep 17 00:00:00 2001 From: Chris Wu Date: Mon, 4 Oct 2021 10:08:57 -0700 Subject: [PATCH 6/7] Fix acceptance tests --- .../connectors/source-jira/source_jira/streams.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-jira/source_jira/streams.py b/airbyte-integrations/connectors/source-jira/source_jira/streams.py index 7deffd22c79e5..2979c244e9301 100644 --- a/airbyte-integrations/connectors/source-jira/source_jira/streams.py +++ b/airbyte-integrations/connectors/source-jira/source_jira/streams.py @@ -376,6 +376,7 @@ class IssuePropertyKeys(JiraStream): """ parse_response_root = "key" + top_level_stream = False def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: key = stream_slice["key"] @@ -708,6 +709,7 @@ class ScreenTabs(JiraStream): https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-screen-tabs/#api-rest-api-3-screens-screenid-tabs-get """ + raise_on_http_errors = False top_level_stream = False def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: @@ -741,7 +743,8 @@ def read_records(self, stream_slice: Optional[Mapping[str, Any]] = None, **kwarg screen_tabs_stream = ScreenTabs(authenticator=self.authenticator, domain=self._domain) for screen in screens_stream.read_records(sync_mode=SyncMode.full_refresh): for tab in screen_tabs_stream.read_tab_records(stream_slice={"screen_id": screen["id"]}, **kwargs): - yield from super().read_records(stream_slice={"screen_id": screen["id"], "tab_id": tab["id"]}, **kwargs) + if id in screen and id in tab: + yield from super().read_records(stream_slice={"screen_id": screen["id"], "tab_id": tab["id"]}, **kwargs) class ScreenSchemes(JiraStream): From 15ad478bd74be31056a7fbee7bec4c481cdf9cf3 Mon Sep 17 00:00:00 2001 From: Chris Wu Date: Mon, 4 Oct 2021 10:13:08 -0700 Subject: [PATCH 7/7] Fix acceptance tests --- .../connectors/source-jira/source_jira/streams.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-jira/source_jira/streams.py b/airbyte-integrations/connectors/source-jira/source_jira/streams.py index 2979c244e9301..d9902e2afe19a 100644 --- a/airbyte-integrations/connectors/source-jira/source_jira/streams.py +++ b/airbyte-integrations/connectors/source-jira/source_jira/streams.py @@ -743,7 +743,7 @@ def read_records(self, stream_slice: Optional[Mapping[str, Any]] = None, **kwarg screen_tabs_stream = ScreenTabs(authenticator=self.authenticator, domain=self._domain) for screen in screens_stream.read_records(sync_mode=SyncMode.full_refresh): for tab in screen_tabs_stream.read_tab_records(stream_slice={"screen_id": screen["id"]}, **kwargs): - if id in screen and id in tab: + if id in tab: # Check for proper tab record since the ScreenTabs stream doesn't throw http errors yield from super().read_records(stream_slice={"screen_id": screen["id"], "tab_id": tab["id"]}, **kwargs)