🐛Source GitLab: Fix syncing branches for the project with Repository …

…feature disabled (#26422)

Add handling blocked repository for branches stream

airbyte-integrations/connectors/source-gitlab/Dockerfile

@@ -13,5 +13,5 @@ COPY main.py ./
 
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=1.1.0
+LABEL io.airbyte.version=1.1.1
 LABEL io.airbyte.name=airbyte/source-gitlab

airbyte-integrations/connectors/source-gitlab/integration_tests/expected_records_with_ids.jsonl

@@ -1,7 +1,7 @@
 {"stream": "groups", "data": {"id": 11329647, "web_url": "https://gitlab.com/groups/new-group-airbute", "name": "New Group Airbute", "path": "new-group-airbute", "description": "", "visibility": "public", "share_with_group_lock": false, "require_two_factor_authentication": false, "two_factor_grace_period": 48, "project_creation_level": "developer", "auto_devops_enabled": null, "subgroup_creation_level": "maintainer", "emails_disabled": null, "mentions_disabled": null, "lfs_enabled": true, "default_branch_protection": 2, "avatar_url": null, "request_access_enabled": true, "full_name": "New Group Airbute", "full_path": "new-group-airbute", "created_at": "2021-03-15T15:55:53.613Z", "parent_id": null, "ldap_cn": null, "ldap_access": null, "wiki_access_level": "enabled", "shared_with_groups": [], "runners_token": "GR1348941-PhosPap-Sf1UxL1g6m4", "prevent_sharing_groups_outside_hierarchy": false, "shared_projects": [], "shared_runners_minutes_limit": null, "extra_shared_runners_minutes_limit": null, "prevent_forking_outside_group": null, "membership_lock": false, "projects": [{"id": 25157276, "path_with_namespace": "new-group-airbute/new-ci-test-project"}]}, "emitted_at": 1683114590269}
 {"stream": "groups", "data": {"id": 61014882, "web_url": "https://gitlab.com/groups/new-group-airbute/test-subgroup-airbyte/test-private-sg", "name": "Test Private SG", "path": "test-private-sg", "description": "", "visibility": "private", "share_with_group_lock": false, "require_two_factor_authentication": false, "two_factor_grace_period": 48, "project_creation_level": "developer", "auto_devops_enabled": null, "subgroup_creation_level": "maintainer", "emails_disabled": null, "mentions_disabled": null, "lfs_enabled": true, "default_branch_protection": 2, "avatar_url": null, "request_access_enabled": true, "full_name": "New Group Airbute / Test Subgroup Airbyte / Test Private SG", "full_path": "new-group-airbute/test-subgroup-airbyte/test-private-sg", "created_at": "2022-12-02T08:46:22.648Z", "parent_id": 61014863, "ldap_cn": null, "ldap_access": null, "wiki_access_level": "enabled", "shared_with_groups": [], "runners_token": "GR1348941bjUaJQy2zzar-JmNBjfq", "shared_projects": [], "shared_runners_minutes_limit": null, "extra_shared_runners_minutes_limit": null, "prevent_forking_outside_group": null, "membership_lock": false, "projects": []}, "emitted_at": 1683114590768}
 {"stream": "groups", "data": {"id": 61015181, "web_url": "https://gitlab.com/groups/new-group-airbute/test-public-sg/test-sg-public-2/test-private-subsubg-1", "name": "Test Private SubSubG 1", "path": "test-private-subsubg-1", "description": "", "visibility": "private", "share_with_group_lock": false, "require_two_factor_authentication": false, "two_factor_grace_period": 48, "project_creation_level": "developer", "auto_devops_enabled": null, "subgroup_creation_level": "maintainer", "emails_disabled": null, "mentions_disabled": null, "lfs_enabled": true, "default_branch_protection": 2, "avatar_url": null, "request_access_enabled": true, "full_name": "New Group Airbute / Test Public SG / Test SG Public 2 / Test Private SubSubG 1", "full_path": "new-group-airbute/test-public-sg/test-sg-public-2/test-private-subsubg-1", "created_at": "2022-12-02T08:54:42.252Z", "parent_id": 61014943, "ldap_cn": null, "ldap_access": null, "wiki_access_level": "enabled", "shared_with_groups": [], "runners_token": "GR1348941x8xQf6K-UvnnyJ-bcut4", "shared_projects": [], "shared_runners_minutes_limit": null, "extra_shared_runners_minutes_limit": null, "prevent_forking_outside_group": null, "membership_lock": false, "projects": [{"id": 41551658, "path_with_namespace": "new-group-airbute/test-public-sg/test-sg-public-2/test-private-subsubg-1/test_project_in_nested_subgroup"}]}, "emitted_at": 1683114591226}
-{"stream": "projects", "data": {"id": 25157276, "description": "", "name": "New CI Test Project ", "name_with_namespace": "New Group Airbute / New CI Test Project ", "path": "new-ci-test-project", "path_with_namespace": "new-group-airbute/new-ci-test-project", "created_at": "2021-03-15T15:08:36.498Z", "default_branch": "master", "tag_list": [], "topics": [], "ssh_url_to_repo": "git@gitlab.com:new-group-airbute/new-ci-test-project.git", "http_url_to_repo": "https://gitlab.com/new-group-airbute/new-ci-test-project.git", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project", "readme_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/blob/master/README.md", "forks_count": 0, "avatar_url": null, "star_count": 0, "last_activity_at": "2022-12-13T09:39:47.235Z", "namespace": {"id": 11329647, "name": "New Group Airbute", "path": "new-group-airbute", "kind": "group", "full_path": "new-group-airbute", "parent_id": null, "avatar_url": null, "web_url": "https://gitlab.com/groups/new-group-airbute"}, "container_registry_image_prefix": "registry.gitlab.com/new-group-airbute/new-ci-test-project", "_links": {"self": "https://gitlab.com/api/v4/projects/25157276", "issues": "https://gitlab.com/api/v4/projects/25157276/issues", "merge_requests": "https://gitlab.com/api/v4/projects/25157276/merge_requests", "repo_branches": "https://gitlab.com/api/v4/projects/25157276/repository/branches", "labels": "https://gitlab.com/api/v4/projects/25157276/labels", "events": "https://gitlab.com/api/v4/projects/25157276/events", "members": "https://gitlab.com/api/v4/projects/25157276/members", "cluster_agents": "https://gitlab.com/api/v4/projects/25157276/cluster_agents"}, "packages_enabled": true, "empty_repo": false, "archived": false, "visibility": "private", "resolve_outdated_diff_discussions": false, "container_expiration_policy": {"cadence": "1d", "enabled": false, "keep_n": 10, "older_than": "90d", "name_regex": ".*", "name_regex_keep": null, "next_run_at": "2021-03-16T15:08:36.518Z"}, "issues_enabled": true, "merge_requests_enabled": true, "wiki_enabled": true, "jobs_enabled": true, "snippets_enabled": true, "container_registry_enabled": true, "service_desk_enabled": true, "service_desk_address": "contact-project+new-group-airbute-new-ci-test-project-25157276-issue-@incoming.gitlab.com", "can_create_merge_request_in": true, "issues_access_level": "enabled", "repository_access_level": "enabled", "merge_requests_access_level": "enabled", "forking_access_level": "enabled", "wiki_access_level": "enabled", "builds_access_level": "enabled", "snippets_access_level": "enabled", "pages_access_level": "private", "analytics_access_level": "enabled", "container_registry_access_level": "enabled", "security_and_compliance_access_level": "private", "releases_access_level": "enabled", "environments_access_level": "enabled", "feature_flags_access_level": "enabled", "infrastructure_access_level": "enabled", "monitor_access_level": "enabled", "emails_disabled": null, "shared_runners_enabled": true, "lfs_enabled": true, "creator_id": 8375961, "import_url": null, "import_type": null, "import_status": "none", "import_error": null, "open_issues_count": 31, "description_html": "", "updated_at": "2022-12-13T09:39:47.235Z", "ci_default_git_depth": 50, "ci_forward_deployment_enabled": true, "ci_job_token_scope_enabled": false, "ci_separated_caches": true, "ci_allow_fork_pipelines_to_run_in_parent_project": true, "build_git_strategy": "fetch", "keep_latest_artifact": true, "restrict_user_defined_variables": false, "runners_token": "GR1348941eMJgWDU69xyyshaNsaTZ", "runner_token_expiration_interval": null, "group_runners_enabled": true, "auto_cancel_pending_pipelines": "enabled", "build_timeout": 3600, "auto_devops_enabled": false, "auto_devops_deploy_strategy": "continuous", "ci_config_path": "", "public_jobs": true, "shared_with_groups": [], "only_allow_merge_if_pipeline_succeeds": false, "allow_merge_on_skipped_pipeline": null, "request_access_enabled": true, "only_allow_merge_if_all_discussions_are_resolved": false, "remove_source_branch_after_merge": true, "printing_merge_request_link_enabled": true, "merge_method": "merge", "squash_option": "default_off", "enforce_auth_checks_on_uploads": true, "suggestion_commit_message": null, "merge_commit_template": null, "squash_commit_template": null, "issue_branch_template": null, "statistics": {"commit_count": 3, "storage_size": 291925, "repository_size": 283115, "wiki_size": 0, "lfs_objects_size": 0, "job_artifacts_size": 8810, "pipeline_artifacts_size": 0, "packages_size": 0, "snippets_size": 0, "uploads_size": 0}, "autoclose_referenced_issues": true, "external_authorization_classification_label": "", "requirements_enabled": false, "requirements_access_level": "enabled", "security_and_compliance_enabled": true, "compliance_frameworks": [], "permissions": {"project_access": {"access_level": 40, "notification_level": 3}, "group_access": {"access_level": 50, "notification_level": 3}}}, "emitted_at": 1684137553535}
+{"stream": "projects", "data": {"id": 25157276, "description": "", "name": "New CI Test Project ", "name_with_namespace": "New Group Airbute / New CI Test Project ", "path": "new-ci-test-project", "path_with_namespace": "new-group-airbute/new-ci-test-project", "created_at": "2021-03-15T15:08:36.498Z", "default_branch": "master", "tag_list": [], "topics": [], "ssh_url_to_repo": "git@gitlab.com:new-group-airbute/new-ci-test-project.git", "http_url_to_repo": "https://gitlab.com/new-group-airbute/new-ci-test-project.git", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project", "readme_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/blob/master/README.md", "forks_count": 0, "avatar_url": null, "star_count": 0, "last_activity_at": "2022-12-13T09:39:47.235Z", "namespace": {"id": 11329647, "name": "New Group Airbute", "path": "new-group-airbute", "kind": "group", "full_path": "new-group-airbute", "parent_id": null, "avatar_url": null, "web_url": "https://gitlab.com/groups/new-group-airbute"}, "container_registry_image_prefix": "registry.gitlab.com/new-group-airbute/new-ci-test-project", "_links": {"self": "https://gitlab.com/api/v4/projects/25157276", "issues": "https://gitlab.com/api/v4/projects/25157276/issues", "merge_requests": "https://gitlab.com/api/v4/projects/25157276/merge_requests", "repo_branches": "https://gitlab.com/api/v4/projects/25157276/repository/branches", "labels": "https://gitlab.com/api/v4/projects/25157276/labels", "events": "https://gitlab.com/api/v4/projects/25157276/events", "members": "https://gitlab.com/api/v4/projects/25157276/members", "cluster_agents": "https://gitlab.com/api/v4/projects/25157276/cluster_agents"}, "packages_enabled": true, "empty_repo": false, "archived": false, "visibility": "private", "resolve_outdated_diff_discussions": false, "container_expiration_policy": {"cadence": "1d", "enabled": false, "keep_n": 10, "older_than": "90d", "name_regex": ".*", "name_regex_keep": null, "next_run_at": "2021-03-16T15:08:36.518Z"}, "issues_enabled": true, "merge_requests_enabled": true, "wiki_enabled": true, "jobs_enabled": true, "snippets_enabled": true, "container_registry_enabled": true, "service_desk_enabled": true, "service_desk_address": "contact-project+new-group-airbute-new-ci-test-project-25157276-issue-@incoming.gitlab.com", "can_create_merge_request_in": true, "issues_access_level": "enabled", "repository_access_level": "private", "merge_requests_access_level": "private", "forking_access_level": "enabled", "wiki_access_level": "enabled", "builds_access_level": "private", "snippets_access_level": "enabled", "pages_access_level": "private", "analytics_access_level": "enabled", "container_registry_access_level": "enabled", "security_and_compliance_access_level": "private", "releases_access_level": "enabled", "environments_access_level": "enabled", "feature_flags_access_level": "enabled", "infrastructure_access_level": "enabled", "monitor_access_level": "enabled", "emails_disabled": false, "shared_runners_enabled": true, "lfs_enabled": true, "creator_id": 8375961, "import_url": null, "import_type": null, "import_status": "none", "import_error": null, "open_issues_count": 31, "description_html": "", "updated_at": "2023-05-23T12:12:18.623Z", "ci_default_git_depth": 50, "ci_forward_deployment_enabled": true, "ci_job_token_scope_enabled": false, "ci_separated_caches": true, "ci_allow_fork_pipelines_to_run_in_parent_project": true, "build_git_strategy": "fetch", "keep_latest_artifact": true, "restrict_user_defined_variables": false, "runners_token": "GR1348941eMJgWDU69xyyshaNsaTZ", "runner_token_expiration_interval": null, "group_runners_enabled": true, "auto_cancel_pending_pipelines": "enabled", "build_timeout": 3600, "auto_devops_enabled": false, "auto_devops_deploy_strategy": "continuous", "ci_config_path": "", "public_jobs": true, "shared_with_groups": [], "only_allow_merge_if_pipeline_succeeds": false, "allow_merge_on_skipped_pipeline": null, "request_access_enabled": true, "only_allow_merge_if_all_discussions_are_resolved": false, "remove_source_branch_after_merge": true, "printing_merge_request_link_enabled": true, "merge_method": "merge", "squash_option": "default_off", "enforce_auth_checks_on_uploads": true, "suggestion_commit_message": null, "merge_commit_template": null, "squash_commit_template": null, "issue_branch_template": null, "statistics": {"commit_count": 3, "storage_size": 291925, "repository_size": 283115, "wiki_size": 0, "lfs_objects_size": 0, "job_artifacts_size": 8810, "pipeline_artifacts_size": 0, "packages_size": 0, "snippets_size": 0, "uploads_size": 0}, "autoclose_referenced_issues": true, "external_authorization_classification_label": "", "requirements_enabled": false, "requirements_access_level": "enabled", "security_and_compliance_enabled": true, "compliance_frameworks": [], "permissions": {"project_access": {"access_level": 40, "notification_level": 3}, "group_access": {"access_level": 50, "notification_level": 3}}}, "emitted_at": 1684137553535}
 {"stream": "branches", "data": {"name": "31-fake-issue-30", "merged": true, "protected": false, "developers_can_push": false, "developers_can_merge": false, "can_push": true, "default": false, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/tree/31-fake-issue-30", "commit_id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "project_id": 25157276}, "emitted_at": 1683114594890}
 {"stream": "branches", "data": {"name": "master", "merged": false, "protected": true, "developers_can_push": false, "developers_can_merge": false, "can_push": true, "default": true, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/tree/master", "commit_id": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "project_id": 25157276}, "emitted_at": 1683114594892}
 {"stream": "branches", "data": {"name": "new-test-branch", "merged": true, "protected": false, "developers_can_push": false, "developers_can_merge": false, "can_push": true, "default": false, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/tree/new-test-branch", "commit_id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "project_id": 25157276}, "emitted_at": 1683114594892}

airbyte-integrations/connectors/source-gitlab/metadata.yaml

@@ -5,7 +5,7 @@ data:
   connectorSubtype: api
   connectorType: source
   definitionId: 5e6175e5-68e1-4c17-bff9-56103bbb0d80
-  dockerImageTag: 1.1.0
+  dockerImageTag: 1.1.1
   dockerRepository: airbyte/source-gitlab
   githubIssueLabel: source-gitlab
   icon: gitlab.svg

airbyte-integrations/connectors/source-gitlab/source_gitlab/streams.py

@@ -22,6 +22,7 @@ class GitlabStream(HttpStream, ABC):
     flatten_id_keys = []
     flatten_list_keys = []
     per_page = 50
+    non_retriable_codes: List[int] = (403,)
 
     def __init__(self, api_url: str, **kwargs):
         super().__init__(**kwargs)
@@ -66,17 +67,17 @@ def availability_strategy(self) -> Optional["AvailabilityStrategy"]:
 
     def should_retry(self, response: requests.Response) -> bool:
         # Gitlab API returns a 403 response in case a feature is disabled in a project (pipelines/jobs for instance).
-        if response.status_code == 403:
+        if response.status_code in self.non_retriable_codes:
             setattr(self, "raise_on_http_errors", False)
             self.logger.warning(
-                f"Got 403 error when accessing URL {response.request.url}."
+                f"Got {response.status_code} error when accessing URL {response.request.url}."
                 f" Very likely the feature is disabled for this project and/or group. Please double check it, or report a bug otherwise."
             )
             return False
         return super().should_retry(response)
 
     def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]:
-        if response.status_code != 200:
+        if response.status_code in self.non_retriable_codes:
             return
         response_data = response.json()
         if isinstance(response_data, dict):
@@ -86,7 +87,7 @@ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str,
             return {"page": self.page}
 
     def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]:
-        if response.status_code == 403:
+        if response.status_code in self.non_retriable_codes:
             return []
         response_data = response.json()
         if isinstance(response_data, list):
@@ -310,6 +311,7 @@ class Branches(GitlabChildStream):
     primary_key = "name"
     flatten_id_keys = ["commit"]
     flatten_parent_id = True
+    non_retriable_codes = (403, 404)
 
 
 class Commits(IncrementalGitlabChildStream):

airbyte-integrations/connectors/source-gitlab/unit_tests/test_streams.py

@@ -6,7 +6,7 @@
 
 import pytest
 from airbyte_cdk.sources.streams.http.auth import NoAuth
-from source_gitlab.streams import Commits, Jobs, MergeRequestCommits, MergeRequests, Pipelines, Projects, Releases, Tags
+from source_gitlab.streams import Branches, Commits, Jobs, MergeRequestCommits, MergeRequests, Pipelines, Projects, Releases, Tags
 
 auth_params = {"authenticator": NoAuth(), "api_url": "gitlab.com"}
 
@@ -19,6 +19,7 @@
 releases = Releases(parent_stream=projects, **auth_params)
 jobs = Jobs(parent_stream=pipelines, **auth_params)
 merge_request_commits = MergeRequestCommits(parent_stream=merge_requests, **auth_params)
+branches = Branches(parent_stream=projects, **auth_params)
 commits = Commits(parent_stream=projects, repository_part=True, start_date=str(start_date), **auth_params)
 
 
@@ -136,3 +137,11 @@ def test_transform(requests_mock, stream, response_mocks, expected_records):
 )
 def test_updated_state(stream, current_state, latest_record, new_state):
     assert stream.get_updated_state(current_state, latest_record) == new_state
+
+
+def test_blocked_branches(requests_mock):
+    requests_mock.get("/api/v4/projects/p_1/branches", status_code=404)
+    for stream_slice in branches.stream_slices(sync_mode="full_refresh"):
+        records = list(branches.read_records(sync_mode="full_refresh", stream_slice=stream_slice))
+    assert records == []
+    assert requests_mock.call_count == 1

docs/integrations/sources/gitlab.md

@@ -105,6 +105,7 @@ Gitlab has the [rate limits](https://docs.gitlab.com/ee/user/gitlab_com/index.ht
 
 | Version | Date       | Pull Request                                             | Subject                                                                                    |
 |:--------|:-----------|:---------------------------------------------------------|:-------------------------------------------------------------------------------------------|
+| 1.1.1   | 2023-05-23 | [26422](https://github.com/airbytehq/airbyte/pull/26422) | Fix error `404 Repository Not Found` when syncing project with Repository feature disabled |
 | 1.1.0   | 2023-05-10 | [25948](https://github.com/airbytehq/airbyte/pull/25948) | Introduce two new fields in the `Projects` stream schema                                   |
 | 1.0.4   | 2023-04-20 | [21373](https://github.com/airbytehq/airbyte/pull/21373) | Accept api_url with or without scheme                                                      |
 | 1.0.3   | 2023-02-14 | [22992](https://github.com/airbytehq/airbyte/pull/22992) | Specified date formatting in specification                                                 |