From 49bee81fc24ee81a22eea0ef2fb8d8fbce3c9425 Mon Sep 17 00:00:00 2001 From: Artem Inzhyyants <36314070+artem1205@users.noreply.github.com> Date: Fri, 6 Jan 2023 19:50:23 +0100 Subject: [PATCH] Source Github: Raise Error if no organizations or repos are available (#21084) * Source Github: Raise Error if no organizations or repos are available * Source GitHub: Update docs; bump version * Source GitHub: Add test * Source GitHub: Exclude empty streams * auto-bump connector version Co-authored-by: Octavia Squidington III --- .../resources/seed/source_definitions.yaml | 2 +- .../src/main/resources/seed/source_specs.yaml | 2 +- .../connectors/source-github/Dockerfile | 2 +- .../source-github/acceptance-test-config.yml | 106 ++++++++++-------- .../source-github/source_github/source.py | 2 + .../source-github/unit_tests/test_source.py | 9 ++ docs/integrations/sources/github.md | 3 +- 7 files changed, 74 insertions(+), 52 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 6a3b23382eed..3f9684b03d1d 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -561,7 +561,7 @@ - name: GitHub sourceDefinitionId: ef69ef6e-aa7f-4af1-a01d-ef775033524e dockerRepository: airbyte/source-github - dockerImageTag: 0.3.10 + dockerImageTag: 0.3.11 documentationUrl: https://docs.airbyte.com/integrations/sources/github icon: github.svg sourceType: api diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 897e585856d0..41c07d46c771 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -4587,7 +4587,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-github:0.3.10" +- dockerImage: "airbyte/source-github:0.3.11" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/github" connectionSpecification: diff --git a/airbyte-integrations/connectors/source-github/Dockerfile b/airbyte-integrations/connectors/source-github/Dockerfile index 782915f127fc..854fd1899f8b 100644 --- a/airbyte-integrations/connectors/source-github/Dockerfile +++ b/airbyte-integrations/connectors/source-github/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.3.10 +LABEL io.airbyte.version=0.3.11 LABEL io.airbyte.name=airbyte/source-github diff --git a/airbyte-integrations/connectors/source-github/acceptance-test-config.yml b/airbyte-integrations/connectors/source-github/acceptance-test-config.yml index f629b322298b..06c296be3cda 100644 --- a/airbyte-integrations/connectors/source-github/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-github/acceptance-test-config.yml @@ -1,55 +1,65 @@ connector_image: airbyte/source-github:dev -tests: +acceptance_tests: spec: - - spec_path: "source_github/spec.json" + tests: + - spec_path: "source_github/spec.json" connection: - - config_path: "secrets/config.json" - status: "succeed" - - config_path: "secrets/config_oauth.json" - status: "succeed" - - config_path: "integration_tests/invalid_config.json" - status: "failed" + tests: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "secrets/config_oauth.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" discovery: - - config_path: "secrets/config.json" - - config_path: "secrets/config_oauth.json" + tests: + - config_path: "secrets/config.json" + - config_path: "secrets/config_oauth.json" basic_read: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: + - name: "events" + bypass_reason: "Only events created within the past 90 days can be showed" incremental: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - future_state_path: "integration_tests/abnormal_state.json" - cursor_paths: - comments: ["airbytehq/integration-test", "updated_at"] - commit_comment_reactions: - ["airbytehq/integration-test", "55538825", "created_at"] - commit_comments: ["airbytehq/integration-test", "updated_at"] - commits: ["airbytehq/integration-test", "master", "created_at"] - deployments: ["airbytehq/integration-test", "updated_at"] - events: ["airbytehq/integration-test", "created_at"] - issue_comment_reactions: - ["airbytehq/integration-test", "907296275", "created_at"] - issue_events: ["airbytehq/integration-test", "created_at"] - issue_milestones: ["airbytehq/integration-test", "updated_at"] - issue_reactions: ["airbytehq/integration-test", "11", "created_at"] - issues: ["airbytehq/integration-test", "updated_at"] - project_cards: - ["airbytehq/integration-test", "13167124", "17807006", "updated_at"] - project_columns: - ["airbytehq/integration-test", "13167124", "updated_at"] - projects: ["airbytehq/integration-test", "updated_at"] - pull_request_comment_reactions: - ["airbytehq/integration-test", "699253726", "created_at"] - pull_request_stats: ["airbytehq/integration-test", "updated_at"] - pull_requests: ["airbytehq/integration-test", "updated_at"] - releases: ["airbytehq/integration-test", "created_at"] - repositories: ["airbytehq", "updated_at"] - review_comments: ["airbytehq/integration-test", "updated_at"] - reviews: ["airbytehq/integration-test", "updated_at"] - stargazers: ["airbytehq/integration-test", "starred_at"] - workflow_runs: ["airbytehq/integration-test", "updated_at"] - workflows: ["airbytehq/integration-test", "updated_at"] - workflow_jobs: ["airbytehq/integration-test", "completed_at"] + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + future_state: + future_state_path: "integration_tests/abnormal_state.json" + cursor_paths: + comments: ["airbytehq/integration-test", "updated_at"] + commit_comment_reactions: + ["airbytehq/integration-test", "55538825", "created_at"] + commit_comments: ["airbytehq/integration-test", "updated_at"] + commits: ["airbytehq/integration-test", "master", "created_at"] + deployments: ["airbytehq/integration-test", "updated_at"] + events: ["airbytehq/integration-test", "created_at"] + issue_comment_reactions: + ["airbytehq/integration-test", "907296275", "created_at"] + issue_events: ["airbytehq/integration-test", "created_at"] + issue_milestones: ["airbytehq/integration-test", "updated_at"] + issue_reactions: ["airbytehq/integration-test", "11", "created_at"] + issues: ["airbytehq/integration-test", "updated_at"] + project_cards: + ["airbytehq/integration-test", "13167124", "17807006", "updated_at"] + project_columns: + ["airbytehq/integration-test", "13167124", "updated_at"] + projects: ["airbytehq/integration-test", "updated_at"] + pull_request_comment_reactions: + ["airbytehq/integration-test", "699253726", "created_at"] + pull_request_stats: ["airbytehq/integration-test", "updated_at"] + pull_requests: ["airbytehq/integration-test", "updated_at"] + releases: ["airbytehq/integration-test", "created_at"] + repositories: ["airbytehq", "updated_at"] + review_comments: ["airbytehq/integration-test", "updated_at"] + reviews: ["airbytehq/integration-test", "updated_at"] + stargazers: ["airbytehq/integration-test", "starred_at"] + workflow_runs: ["airbytehq/integration-test", "updated_at"] + workflows: ["airbytehq/integration-test", "updated_at"] + workflow_jobs: ["airbytehq/integration-test", "completed_at"] full_refresh: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-github/source_github/source.py b/airbyte-integrations/connectors/source-github/source_github/source.py index db18092f14c7..eeac7c17abca 100644 --- a/airbyte-integrations/connectors/source-github/source_github/source.py +++ b/airbyte-integrations/connectors/source-github/source_github/source.py @@ -176,6 +176,8 @@ def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> def streams(self, config: Mapping[str, Any]) -> List[Stream]: authenticator = self._get_authenticator(config) organizations, repositories = self._get_org_repositories(config=config, authenticator=authenticator) + if not any((organizations, repositories)): + raise Exception("No streams available. Please check permissions") page_size = config.get("page_size_for_large_streams", DEFAULT_PAGE_SIZE_FOR_LARGE_STREAM) organization_args = {"authenticator": authenticator, "organizations": organizations} diff --git a/airbyte-integrations/connectors/source-github/unit_tests/test_source.py b/airbyte-integrations/connectors/source-github/unit_tests/test_source.py index 72070259e5f4..6a089de64654 100644 --- a/airbyte-integrations/connectors/source-github/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-github/unit_tests/test_source.py @@ -127,3 +127,12 @@ def test_get_org_repositories(): assert set(repositories) == {"airbytehq/integration-test", "docker/docker-py", "docker/compose"} assert set(organisations) == {"airbytehq", "docker"} + + +def test_organization_or_repo_available(): + SourceGithub._get_org_repositories = MagicMock(return_value=(False, False)) + source = SourceGithub() + with pytest.raises(Exception) as exc_info: + config = {"access_token": "test_token", "repository": ""} + source.streams(config=config) + assert exc_info.value.args[0] == "No streams available. Please check permissions" diff --git a/docs/integrations/sources/github.md b/docs/integrations/sources/github.md index 017d83aaa192..96ba9aebac5d 100644 --- a/docs/integrations/sources/github.md +++ b/docs/integrations/sources/github.md @@ -162,7 +162,8 @@ The GitHub connector should not run into GitHub API limitations under normal usa ## Changelog | Version | Date | Pull Request | Subject | -| :------ | :--------- | :---------------------------------------------------------------------------------------------------------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +|:--------|:-----------|:------------------------------------------------------------------------------------------------------------------|:--------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| 0.3.11 | 2023-01-06 | [21084](https://github.com/airbytehq/airbyte/pull/21084) | Raise Error if no organizations or repos are available during read | | 0.3.10 | 2022-12-15 | [20523](https://github.com/airbytehq/airbyte/pull/20523) | Revert changes from 0.3.9 | | 0.3.9 | 2022-12-14 | [19978](https://github.com/airbytehq/airbyte/pull/19978) | Update CDK dependency; move custom HTTPError handling into `AvailabilityStrategy` classes | | 0.3.8 | 2022-11-10 | [19299](https://github.com/airbytehq/airbyte/pull/19299) | Fix events and workflow_runs datetimes |