From 69f29f80fcff40672f7064ec4c445232e70daac2 Mon Sep 17 00:00:00 2001 From: Pages Coffie Date: Tue, 17 Sep 2024 19:58:36 +0000 Subject: [PATCH 1/8] improve logic of handling gitops --- .../gitlab_integration/core/entities.py | 8 +- .../events/event_handler.py | 2 +- .../gitlab_integration/events/hooks/push.py | 78 ++++++++++++++----- .../gitlab_integration/gitlab_service.py | 23 +++--- .../gitlab/gitlab_integration/ocean.py | 4 +- 5 files changed, 80 insertions(+), 35 deletions(-) diff --git a/integrations/gitlab/gitlab_integration/core/entities.py b/integrations/gitlab/gitlab_integration/core/entities.py index 6bfe530416..04fe56b3d6 100644 --- a/integrations/gitlab/gitlab_integration/core/entities.py +++ b/integrations/gitlab/gitlab_integration/core/entities.py @@ -5,19 +5,23 @@ from port_ocean.core.models import Entity +from gitlab_integration.core.async_fetcher import AsyncFetcher + FILE_PROPERTY_PREFIX = "file://" SEARCH_PROPERTY_PREFIX = "search://" JSON_SUFFIX = ".json" -def generate_entity_from_port_yaml( +async def generate_entity_from_port_yaml( raw_entity: Entity, project: Project, ref: str ) -> Entity: properties = {} for key, value in raw_entity.properties.items(): if isinstance(value, str) and value.startswith(FILE_PROPERTY_PREFIX): file_meta = Path(value.replace(FILE_PROPERTY_PREFIX, "")) - gitlab_file = project.files.get(file_path=str(file_meta), ref=ref) + gitlab_file = await AsyncFetcher.fetch_single( + project.files.get, str(file_meta), ref + ) if file_meta.suffix == JSON_SUFFIX: properties[key] = json.loads(gitlab_file.decode().decode("utf-8")) diff --git a/integrations/gitlab/gitlab_integration/events/event_handler.py b/integrations/gitlab/gitlab_integration/events/event_handler.py index 60fe2608b0..5e087540df 100644 --- a/integrations/gitlab/gitlab_integration/events/event_handler.py +++ b/integrations/gitlab/gitlab_integration/events/event_handler.py @@ -71,7 +71,7 @@ async def _notify(self, event_id: str, body: dict[str, Any]) -> None: ) if not observers: - logger.debug( + logger.info( f"event: {event_id} has no matching handler. the handlers available are for events: {self._observers.keys()}" ) diff --git a/integrations/gitlab/gitlab_integration/events/hooks/push.py b/integrations/gitlab/gitlab_integration/events/hooks/push.py index d4a32d7259..279f810567 100644 --- a/integrations/gitlab/gitlab_integration/events/hooks/push.py +++ b/integrations/gitlab/gitlab_integration/events/hooks/push.py @@ -4,7 +4,7 @@ from loguru import logger from gitlab.v4.objects import Project -from gitlab_integration.core.utils import generate_ref +from gitlab_integration.core.utils import generate_ref, does_pattern_apply from gitlab_integration.events.hooks.base import ProjectHandler from gitlab_integration.git_integration import GitlabPortAppConfig from gitlab_integration.utils import ObjectKind @@ -26,6 +26,19 @@ async def _on_hook(self, body: dict[str, Any], gitlab_project: Project) -> None: "Invalid push hook. Missing one or more of the required fields (before, after, ref)" ) + added_files = [ + added_file + for commit in body.get("commits", []) + for added_file in commit.get("added", []) + ] + modified_files = [ + modified_file + for commit in body.get("commits", []) + for modified_file in commit.get("modified", []) + ] + + changed_files = list(set(added_files + modified_files)) + config: GitlabPortAppConfig = typing.cast( GitlabPortAppConfig, event.port_app_config ) @@ -33,27 +46,56 @@ async def _on_hook(self, body: dict[str, Any], gitlab_project: Project) -> None: branch = config.branch or gitlab_project.default_branch if generate_ref(branch) == ref: - entities_before, entities_after = ( - await self.gitlab_service.get_entities_diff( - gitlab_project, config.spec_path, before, after, branch - ) + spec_path = config.spec_path + if not isinstance(spec_path, list): + spec_path = [spec_path] + await self._process_changed_files( + gitlab_project, changed_files, spec_path, before, after, branch ) - # update the entities diff found in the `config.spec_path` file the user configured - await ocean.update_diff( - {"before": entities_before, "after": entities_after}, - UserAgentType.gitops, - ) - # update information regarding the project as well - logger.info( - f"Updating project information after push hook for project {gitlab_project.path_with_namespace}" - ) - enriched_project = await self.gitlab_service.enrich_project_with_extras( - gitlab_project - ) - await ocean.register_raw(ObjectKind.PROJECT, [enriched_project]) else: logger.debug( f"Skipping push hook for project {gitlab_project.path_with_namespace} because the ref {ref} " f"does not match the branch {branch}" ) + + async def _process_changed_files( + self, + gitlab_project: Project, + changed_files: list[str], + spec_path: list[str], + before: str, + after: str, + branch: str, + ) -> None: + logger.info( + f"Processing changed files {changed_files} for project {gitlab_project.path_with_namespace}" + ) + + for file in changed_files: + if does_pattern_apply(spec_path, file): + logger.info( + f"Found file {file} in spec_path {spec_path} pattern, processing its entity diff" + ) + + entities_before, entities_after = ( + await self.gitlab_service.get_entities_diff( + gitlab_project, file, before, after, branch + ) + ) + + # update the entities diff found in the `config.spec_path` file the user configured + await ocean.update_diff( + {"before": entities_before, "after": entities_after}, + UserAgentType.gitops, + ) + # update information regarding the project as well + logger.info( + f"Updating project information after push hook for project {gitlab_project.path_with_namespace}" + ) + enriched_project = await self.gitlab_service.enrich_project_with_extras( + gitlab_project + ) + await ocean.register_raw(ObjectKind.PROJECT, [enriched_project]) + else: + logger.info(f"Skipping file {file} as it does not match the spec_path pattern {spec_path}") \ No newline at end of file diff --git a/integrations/gitlab/gitlab_integration/gitlab_service.py b/integrations/gitlab/gitlab_integration/gitlab_service.py index 2b94897662..b5fb05ce6b 100644 --- a/integrations/gitlab/gitlab_integration/gitlab_service.py +++ b/integrations/gitlab/gitlab_integration/gitlab_service.py @@ -166,11 +166,14 @@ async def search_files_in_project( if files_with_content: yield files_with_content - def _get_entities_from_git( - self, project: Project, file_name: str, sha: str, ref: str + async def _get_entities_from_git( + self, project: Project, file_path: str | List[str], sha: str, ref: str ) -> List[Entity]: try: - file_content = project.files.get(file_path=file_name, ref=sha) + file_content = await AsyncFetcher.fetch_single( + project.files.get, file_path, sha + ) + entities = yaml.safe_load(file_content.decode()) raw_entities = [ Entity(**entity_data) @@ -179,29 +182,25 @@ def _get_entities_from_git( ) ] return [ - generate_entity_from_port_yaml(entity_data, project, ref) + await generate_entity_from_port_yaml(entity_data, project, ref) for entity_data in raw_entities ] except ParserError as exec: logger.error( - f"Failed to parse gitops entities from gitlab project {project.path_with_namespace},z file {file_name}." + f"Failed to parse gitops entities from gitlab project {project.path_with_namespace},z file {file_path}." f"\n {exec}" ) except Exception: logger.error( - f"Failed to get gitops entities from gitlab project {project.path_with_namespace}, file {file_name}" + f"Failed to get gitops entities from gitlab project {project.path_with_namespace}, file {file_path}" ) return [] async def _get_entities_by_commit( self, project: Project, spec: str | List["str"], commit: str, ref: str ) -> List[Entity]: - spec_paths = await self.get_all_file_paths(project, spec, commit) - return [ - entity - for path in spec_paths - for entity in self._get_entities_from_git(project, path, commit, ref) - ] + + return await self._get_entities_from_git(project, spec, commit, ref) def should_run_for_path(self, path: str) -> bool: return any(does_pattern_apply(mapping, path) for mapping in self.group_mapping) diff --git a/integrations/gitlab/gitlab_integration/ocean.py b/integrations/gitlab/gitlab_integration/ocean.py index e10fb92224..12e5e9949f 100644 --- a/integrations/gitlab/gitlab_integration/ocean.py +++ b/integrations/gitlab/gitlab_integration/ocean.py @@ -32,7 +32,7 @@ async def handle_webhook_request(group_id: str, request: Request) -> dict[str, A event_id = f"{request.headers.get('X-Gitlab-Event')}:{group_id}" with logger.contextualize(event_id=event_id): try: - logger.debug(f"Received webhook event {event_id} from Gitlab") + logger.info(f"Received webhook event {event_id} from Gitlab") body = await request.json() await event_handler.notify(event_id, body) return {"ok": True} @@ -50,7 +50,7 @@ async def handle_system_webhook_request(request: Request) -> dict[str, Any]: # some system hooks have event_type instead of event_name in the body, such as merge_request events event_name: str = str(body.get("event_name") or body.get("event_type")) with logger.contextualize(event_name=event_name): - logger.debug(f"Received system webhook event {event_name} from Gitlab") + logger.info(f"Received system webhook event {event_name} from Gitlab") await system_event_handler.notify(event_name, body) return {"ok": True} From ad772f14a11563e01cea8e6c8d02cde9dd3460f4 Mon Sep 17 00:00:00 2001 From: Pages Coffie Date: Tue, 17 Sep 2024 19:59:03 +0000 Subject: [PATCH 2/8] bump integration version --- integrations/gitlab/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integrations/gitlab/pyproject.toml b/integrations/gitlab/pyproject.toml index 2af63ba5c9..4efaf60003 100644 --- a/integrations/gitlab/pyproject.toml +++ b/integrations/gitlab/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "gitlab" -version = "0.1.120" +version = "0.1.121" description = "Gitlab integration for Port using Port-Ocean Framework" authors = ["Yair Siman-Tov "] From 4796f317cbb7650d7333bf5a8b54bd890f1b16ec Mon Sep 17 00:00:00 2001 From: Pages Coffie Date: Tue, 17 Sep 2024 20:01:24 +0000 Subject: [PATCH 3/8] create changelog --- integrations/gitlab/CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/integrations/gitlab/CHANGELOG.md b/integrations/gitlab/CHANGELOG.md index d9e29c49e7..27da329bd7 100644 --- a/integrations/gitlab/CHANGELOG.md +++ b/integrations/gitlab/CHANGELOG.md @@ -7,6 +7,14 @@ this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm +0.1.121 (2024-09-17) +==================== + +### Improvements + +- Improved on the way the integration handles GitOps push events by using only files that have been changed in the push even rather than fetching the entire repository tree (0.1.121) + + 0.1.120 (2024-09-17) ==================== From 64f571054d8dee77ca88bcb0dcf4855410846779 Mon Sep 17 00:00:00 2001 From: Pages Coffie Date: Tue, 17 Sep 2024 20:09:19 +0000 Subject: [PATCH 4/8] make lint --- integrations/gitlab/gitlab_integration/events/hooks/push.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/integrations/gitlab/gitlab_integration/events/hooks/push.py b/integrations/gitlab/gitlab_integration/events/hooks/push.py index 279f810567..e5fef0cc48 100644 --- a/integrations/gitlab/gitlab_integration/events/hooks/push.py +++ b/integrations/gitlab/gitlab_integration/events/hooks/push.py @@ -98,4 +98,6 @@ async def _process_changed_files( ) await ocean.register_raw(ObjectKind.PROJECT, [enriched_project]) else: - logger.info(f"Skipping file {file} as it does not match the spec_path pattern {spec_path}") \ No newline at end of file + logger.info( + f"Skipping file {file} as it does not match the spec_path pattern {spec_path}" + ) From 40b09e787f1db61a502e1acb2be6622ae5efd891 Mon Sep 17 00:00:00 2001 From: Pages Coffie Date: Wed, 18 Sep 2024 11:43:04 +0000 Subject: [PATCH 5/8] improve logic for gitops and handle delete files --- .../gitlab_integration/events/hooks/push.py | 147 +++++++++++++----- .../gitlab_integration/gitlab_service.py | 4 +- 2 files changed, 114 insertions(+), 37 deletions(-) diff --git a/integrations/gitlab/gitlab_integration/events/hooks/push.py b/integrations/gitlab/gitlab_integration/events/hooks/push.py index e5fef0cc48..81760f86e9 100644 --- a/integrations/gitlab/gitlab_integration/events/hooks/push.py +++ b/integrations/gitlab/gitlab_integration/events/hooks/push.py @@ -1,5 +1,6 @@ import typing from typing import Any +from enum import StrEnum from loguru import logger from gitlab.v4.objects import Project @@ -14,14 +15,24 @@ from port_ocean.context.ocean import ocean +class FileAction(StrEnum): + DELETED = "deleted" + ADDED = "added" + MODIFIED = "modified" + + class PushHook(ProjectHandler): events = ["Push Hook"] system_events = ["push"] async def _on_hook(self, body: dict[str, Any], gitlab_project: Project) -> None: - before, after, ref = body.get("before"), body.get("after"), body.get("ref") + commit_before, commit_after, ref = ( + body.get("before"), + body.get("after"), + body.get("ref"), + ) - if before is None or after is None or ref is None: + if commit_before is None or commit_after is None or ref is None: raise ValueError( "Invalid push hook. Missing one or more of the required fields (before, after, ref)" ) @@ -37,7 +48,11 @@ async def _on_hook(self, body: dict[str, Any], gitlab_project: Project) -> None: for modified_file in commit.get("modified", []) ] - changed_files = list(set(added_files + modified_files)) + removed_files = [ + removed_file + for commit in body.get("commits", []) + for removed_file in commit.get("removed", []) + ] config: GitlabPortAppConfig = typing.cast( GitlabPortAppConfig, event.port_app_config @@ -49,9 +64,43 @@ async def _on_hook(self, body: dict[str, Any], gitlab_project: Project) -> None: spec_path = config.spec_path if not isinstance(spec_path, list): spec_path = [spec_path] - await self._process_changed_files( - gitlab_project, changed_files, spec_path, before, after, branch + + await self._process_files( + gitlab_project, + removed_files, + spec_path, + commit_before, + "", + branch, + FileAction.DELETED, + ) + await self._process_files( + gitlab_project, + added_files, + spec_path, + "", + commit_after, + branch, + FileAction.ADDED, + ) + await self._process_files( + gitlab_project, + modified_files, + spec_path, + commit_before, + commit_after, + branch, + FileAction.MODIFIED, + ) + + # update information regarding the project as well + logger.info( + f"Updating project information after push hook for project {gitlab_project.path_with_namespace}" + ) + enriched_project = await self.gitlab_service.enrich_project_with_extras( + gitlab_project ) + await ocean.register_raw(ObjectKind.PROJECT, [enriched_project]) else: logger.debug( @@ -59,45 +108,71 @@ async def _on_hook(self, body: dict[str, Any], gitlab_project: Project) -> None: f"does not match the branch {branch}" ) - async def _process_changed_files( + async def _process_files( self, gitlab_project: Project, - changed_files: list[str], + files: list[str], spec_path: list[str], - before: str, - after: str, + commit_before: str, + commit_after: str, branch: str, + file_action: FileAction, ) -> None: + if not files: + return logger.info( - f"Processing changed files {changed_files} for project {gitlab_project.path_with_namespace}" + f"Processing {file_action} files {files} for project {gitlab_project.path_with_namespace}" ) - for file in changed_files: - if does_pattern_apply(spec_path, file): - logger.info( - f"Found file {file} in spec_path {spec_path} pattern, processing its entity diff" - ) - - entities_before, entities_after = ( - await self.gitlab_service.get_entities_diff( - gitlab_project, file, before, after, branch + for file in files: + try: + if does_pattern_apply(spec_path, file): + logger.info( + f"Found file {file} in spec_path {spec_path} pattern, processing its entity diff" ) - ) - # update the entities diff found in the `config.spec_path` file the user configured - await ocean.update_diff( - {"before": entities_before, "after": entities_after}, - UserAgentType.gitops, - ) - # update information regarding the project as well - logger.info( - f"Updating project information after push hook for project {gitlab_project.path_with_namespace}" - ) - enriched_project = await self.gitlab_service.enrich_project_with_extras( - gitlab_project - ) - await ocean.register_raw(ObjectKind.PROJECT, [enriched_project]) - else: - logger.info( - f"Skipping file {file} as it does not match the spec_path pattern {spec_path}" + if file_action == FileAction.DELETED: + entities_before = ( + await self.gitlab_service._get_entities_by_commit( + gitlab_project, file, commit_before, branch + ) + ) + await ocean.update_diff( + {"before": entities_before, "after": []}, + UserAgentType.gitops, + ) + + elif file_action == FileAction.ADDED: + entities_after = ( + await self.gitlab_service._get_entities_by_commit( + gitlab_project, file, commit_after, branch + ) + ) + await ocean.update_diff( + {"before": [], "after": entities_after}, + UserAgentType.gitops, + ) + + elif file_action == FileAction.MODIFIED: + entities_before = ( + await self.gitlab_service._get_entities_by_commit( + gitlab_project, file, commit_before, branch + ) + ) + entities_after = ( + await self.gitlab_service._get_entities_by_commit( + gitlab_project, file, commit_after, branch + ) + ) + await ocean.update_diff( + {"before": entities_before, "after": entities_after}, + UserAgentType.gitops, + ) + else: + logger.info( + f"Skipping file {file} as it does not match the spec_path pattern {spec_path}" + ) + except Exception as e: + logger.error( + f"Error processing file {file} in action {file_action}: {str(e)}" ) diff --git a/integrations/gitlab/gitlab_integration/gitlab_service.py b/integrations/gitlab/gitlab_integration/gitlab_service.py index b5fb05ce6b..db43ef9310 100644 --- a/integrations/gitlab/gitlab_integration/gitlab_service.py +++ b/integrations/gitlab/gitlab_integration/gitlab_service.py @@ -199,7 +199,9 @@ async def _get_entities_from_git( async def _get_entities_by_commit( self, project: Project, spec: str | List["str"], commit: str, ref: str ) -> List[Entity]: - + logger.info( + f"Getting entities for project {project.path_with_namespace} in path {spec} at commit {commit} and ref {ref}" + ) return await self._get_entities_from_git(project, spec, commit, ref) def should_run_for_path(self, path: str) -> bool: From de16d81d103a80db8d1513c94784ec2067bf3a70 Mon Sep 17 00:00:00 2001 From: Pages Coffie Date: Wed, 18 Sep 2024 12:17:55 +0000 Subject: [PATCH 6/8] remove info logs and use align with commit events --- .../gitlab_integration/events/hooks/push.py | 82 +++++++++---------- 1 file changed, 40 insertions(+), 42 deletions(-) diff --git a/integrations/gitlab/gitlab_integration/events/hooks/push.py b/integrations/gitlab/gitlab_integration/events/hooks/push.py index 81760f86e9..ca9a0621f9 100644 --- a/integrations/gitlab/gitlab_integration/events/hooks/push.py +++ b/integrations/gitlab/gitlab_integration/events/hooks/push.py @@ -16,7 +16,7 @@ class FileAction(StrEnum): - DELETED = "deleted" + REMOVED = "removed" ADDED = "added" MODIFIED = "modified" @@ -40,18 +40,18 @@ async def _on_hook(self, body: dict[str, Any], gitlab_project: Project) -> None: added_files = [ added_file for commit in body.get("commits", []) - for added_file in commit.get("added", []) + for added_file in commit.get(FileAction.ADDED, []) ] modified_files = [ modified_file for commit in body.get("commits", []) - for modified_file in commit.get("modified", []) + for modified_file in commit.get(FileAction.MODIFIED, []) ] removed_files = [ removed_file for commit in body.get("commits", []) - for removed_file in commit.get("removed", []) + for removed_file in commit.get(FileAction.REMOVED, []) ] config: GitlabPortAppConfig = typing.cast( @@ -72,7 +72,7 @@ async def _on_hook(self, body: dict[str, Any], gitlab_project: Project) -> None: commit_before, "", branch, - FileAction.DELETED, + FileAction.REMOVED, ) await self._process_files( gitlab_project, @@ -127,49 +127,47 @@ async def _process_files( for file in files: try: if does_pattern_apply(spec_path, file): - logger.info( - f"Found file {file} in spec_path {spec_path} pattern, processing its entity diff" - ) - if file_action == FileAction.DELETED: - entities_before = ( - await self.gitlab_service._get_entities_by_commit( - gitlab_project, file, commit_before, branch + match file_action: + case FileAction.REMOVED: + entities_before = ( + await self.gitlab_service._get_entities_by_commit( + gitlab_project, file, commit_before, branch + ) + ) + await ocean.update_diff( + {"before": entities_before, "after": []}, + UserAgentType.gitops, + ) + + case FileAction.ADDED: + entities_after = ( + await self.gitlab_service._get_entities_by_commit( + gitlab_project, file, commit_after, branch + ) + ) + await ocean.update_diff( + {"before": [], "after": entities_after}, + UserAgentType.gitops, ) - ) - await ocean.update_diff( - {"before": entities_before, "after": []}, - UserAgentType.gitops, - ) - - elif file_action == FileAction.ADDED: - entities_after = ( - await self.gitlab_service._get_entities_by_commit( - gitlab_project, file, commit_after, branch + + case FileAction.MODIFIED: + entities_before = ( + await self.gitlab_service._get_entities_by_commit( + gitlab_project, file, commit_before, branch + ) ) - ) - await ocean.update_diff( - {"before": [], "after": entities_after}, - UserAgentType.gitops, - ) - - elif file_action == FileAction.MODIFIED: - entities_before = ( - await self.gitlab_service._get_entities_by_commit( - gitlab_project, file, commit_before, branch + entities_after = ( + await self.gitlab_service._get_entities_by_commit( + gitlab_project, file, commit_after, branch + ) ) - ) - entities_after = ( - await self.gitlab_service._get_entities_by_commit( - gitlab_project, file, commit_after, branch + await ocean.update_diff( + {"before": entities_before, "after": entities_after}, + UserAgentType.gitops, ) - ) - await ocean.update_diff( - {"before": entities_before, "after": entities_after}, - UserAgentType.gitops, - ) else: - logger.info( + logger.debug( f"Skipping file {file} as it does not match the spec_path pattern {spec_path}" ) except Exception as e: From 1cc82ddd36f63a987065a56d52eed8ba06edd833 Mon Sep 17 00:00:00 2001 From: Pages Coffie Date: Wed, 18 Sep 2024 16:12:47 +0000 Subject: [PATCH 7/8] re organize gitops logic for matching and non matching files --- .../gitlab_integration/events/hooks/push.py | 28 ++++++++++++------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/integrations/gitlab/gitlab_integration/events/hooks/push.py b/integrations/gitlab/gitlab_integration/events/hooks/push.py index ca9a0621f9..3465118526 100644 --- a/integrations/gitlab/gitlab_integration/events/hooks/push.py +++ b/integrations/gitlab/gitlab_integration/events/hooks/push.py @@ -123,11 +123,19 @@ async def _process_files( logger.info( f"Processing {file_action} files {files} for project {gitlab_project.path_with_namespace}" ) + matching_files = [file for file in files if does_pattern_apply(spec_path, file)] - for file in files: - try: - if does_pattern_apply(spec_path, file): + if not matching_files: + logger.info("No matching files found for mapping") + logger.debug(f"Files {files} didn't match {spec_path} patten") + return + else: + logger.info( + f"While processing {file_action} Found {len(matching_files)} that matches {spec_path}, matching files: {matching_files}" + ) + for file in matching_files: + try: match file_action: case FileAction.REMOVED: entities_before = ( @@ -166,11 +174,11 @@ async def _process_files( {"before": entities_before, "after": entities_after}, UserAgentType.gitops, ) - else: - logger.debug( - f"Skipping file {file} as it does not match the spec_path pattern {spec_path}" + except Exception as e: + logger.error( + f"Error processing file {file} in action {file_action}: {str(e)}" ) - except Exception as e: - logger.error( - f"Error processing file {file} in action {file_action}: {str(e)}" - ) + skipped_files = set(files) - set(matching_files) + logger.debug( + f"Skipped {len(skipped_files)} files as they didn't match {spec_path} Skipped files: {skipped_files}" + ) From 2c145328a2e97b3cf8784f726d5c64b7bac6e090 Mon Sep 17 00:00:00 2001 From: Pages Coffie Date: Wed, 18 Sep 2024 16:28:49 +0000 Subject: [PATCH 8/8] add logs and try catch to queuing for verbosity --- .../gitlab/gitlab_integration/events/event_handler.py | 6 ++++++ .../gitlab/gitlab_integration/events/hooks/base.py | 9 +++++++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/integrations/gitlab/gitlab_integration/events/event_handler.py b/integrations/gitlab/gitlab_integration/events/event_handler.py index 5e087540df..ce2664f3a4 100644 --- a/integrations/gitlab/gitlab_integration/events/event_handler.py +++ b/integrations/gitlab/gitlab_integration/events/event_handler.py @@ -28,11 +28,16 @@ async def _start_event_processor(self) -> None: logger.info(f"Started {self.__class__.__name__} worker") while True: event_ctx, event_id, body = await self.webhook_tasks_queue.get() + logger.debug(f"Retrieved event: {event_id} from Queue, notifying observers") try: async with event_context( "gitlab_http_event_async_worker", parent_override=event_ctx ): await self._notify(event_id, body) + except Exception as e: + logger.error( + f"Error notifying observers for event: {event_id}, error: {e}" + ) finally: self.webhook_tasks_queue.task_done() @@ -44,6 +49,7 @@ async def _notify(self, event_id: str, body: dict[str, Any]) -> None: pass async def notify(self, event_id: str, body: dict[str, Any]) -> None: + logger.debug(f"Received event: {event_id}, putting it in Queue for processing") await self.webhook_tasks_queue.put( ( deepcopy(current_event_context), diff --git a/integrations/gitlab/gitlab_integration/events/hooks/base.py b/integrations/gitlab/gitlab_integration/events/hooks/base.py index 6ed43a3403..8b2b8a3bcf 100644 --- a/integrations/gitlab/gitlab_integration/events/hooks/base.py +++ b/integrations/gitlab/gitlab_integration/events/hooks/base.py @@ -34,8 +34,13 @@ async def on_hook(self, event: str, body: dict[str, Any]) -> None: logger.info( f"Handling hook {event} for project {project.path_with_namespace}" ) - await self._on_hook(body, project) - logger.info(f"Finished handling {event}") + try: + await self._on_hook(body, project) + logger.info(f"Finished handling {event}") + except Exception as e: + logger.error( + f"Error handling hook {event} for project {project.path_with_namespace}. Error: {e}" + ) else: logger.info( f"Project {body['project']['id']} was filtered for event {event}. Skipping..."