From 9c8bc6c86a43f9e49ea9a4db4af2f9bea9893e58 Mon Sep 17 00:00:00 2001 From: mrT23 Date: Sun, 9 Jun 2024 14:29:32 +0300 Subject: [PATCH 1/4] Update PR review prompts and terminology for clarity and consistency --- pr_agent/algo/utils.py | 9 +++++---- pr_agent/git_providers/github_provider.py | 4 ++-- pr_agent/settings/configuration.toml | 2 +- pr_agent/settings/pr_reviewer_prompts.toml | 8 ++++---- pr_agent/tools/pr_reviewer.py | 11 ++++++++--- tests/unittest/test_convert_to_markdown.py | 2 +- 6 files changed, 21 insertions(+), 15 deletions(-) diff --git a/pr_agent/algo/utils.py b/pr_agent/algo/utils.py index 8c6bf6734..7162c1ae2 100644 --- a/pr_agent/algo/utils.py +++ b/pr_agent/algo/utils.py @@ -39,7 +39,7 @@ def emphasize_header(text: str) -> str: # Splitting the string and wrapping the first part in tags if colon_position != -1: # Everything before the colon (inclusive) is wrapped in tags - transformed_string = "" + text[:colon_position + 1] + "" + text[colon_position + 1:] + transformed_string = "" + text[:colon_position + 1] + "" +'
' + text[colon_position + 1:] else: # If there's no ": ", return the original string transformed_string = text @@ -74,6 +74,7 @@ def convert_to_markdown(output_data: dict, gfm_supported: bool = True, increment emojis = { "Can be split": "๐Ÿ”€", "Possible issues": "โšก", + "Key issues to review": "โšก", "Score": "๐Ÿ…", "Relevant tests": "๐Ÿงช", "Focused PR": "โœจ", @@ -85,9 +86,9 @@ def convert_to_markdown(output_data: dict, gfm_supported: bool = True, increment } markdown_text = "" if not incremental_review: - markdown_text += f"## PR Review ๐Ÿ”\n\n" + markdown_text += f"## PR Reviewer Guide ๐Ÿ”\n\n" else: - markdown_text += f"## Incremental PR Review ๐Ÿ”\n\n" + markdown_text += f"## Incremental PR Reviewer Guide ๐Ÿ”\n\n" markdown_text += f"โฎ๏ธ Review for commits since previous PR-Agent review {incremental_review}.\n\n" if gfm_supported: markdown_text += "\n\n" @@ -110,7 +111,7 @@ def convert_to_markdown(output_data: dict, gfm_supported: bool = True, increment markdown_text += f"\n" elif 'can be split' in key_nice.lower(): markdown_text += process_can_be_split(emoji, value) - elif 'possible issues' in key_nice.lower(): + elif 'key issues to review' in key_nice.lower(): value = value.strip() issues = value.split('\n- ') for i, _ in enumerate(issues): diff --git a/pr_agent/git_providers/github_provider.py b/pr_agent/git_providers/github_provider.py index 99b8521b6..f2be54a3e 100644 --- a/pr_agent/git_providers/github_provider.py +++ b/pr_agent/git_providers/github_provider.py @@ -92,9 +92,9 @@ def get_previous_review(self, *, full: bool, incremental: bool): self.comments = list(self.pr.get_issue_comments()) prefixes = [] if full: - prefixes.append("## PR Review") + prefixes.append("## PR Reviewer Guide") if incremental: - prefixes.append("## Incremental PR Review") + prefixes.append("## Incremental PR Reviewer Guide") for index in range(len(self.comments) - 1, -1, -1): if any(self.comments[index].body.startswith(prefix) for prefix in prefixes): return self.comments[index] diff --git a/pr_agent/settings/configuration.toml b/pr_agent/settings/configuration.toml index 5051ee06a..0fa850133 100644 --- a/pr_agent/settings/configuration.toml +++ b/pr_agent/settings/configuration.toml @@ -33,7 +33,7 @@ require_security_review=true require_soc2_ticket=false soc2_ticket_prompt="Does the PR description include a link to ticket in a project management system (e.g., Jira, Asana, Trello, etc.) ?" # general options -num_code_suggestions=4 +num_code_suggestions=0 inline_code_comments = false ask_and_reflect=false #automatic_review=true diff --git a/pr_agent/settings/pr_reviewer_prompts.toml b/pr_agent/settings/pr_reviewer_prompts.toml index d7d06f419..9f4d73f10 100644 --- a/pr_agent/settings/pr_reviewer_prompts.toml +++ b/pr_agent/settings/pr_reviewer_prompts.toml @@ -57,7 +57,7 @@ class SubPR(BaseModel): class Review(BaseModel): {%- if require_estimate_effort_to_review %} - estimated_effort_to_review_[1-5]: str = Field(description="Estimate, on a scale of 1-5 (inclusive), the time and effort required to review this PR by an experienced and knowledgeable developer. 1 means short and easy review , 5 means long and hard review. Take into account the size, complexity, quality, and the needed changes of the PR code diff. Explain your answer in a short and concise manner.") + estimated_effort_to_review_[1-5]: int = Field(description="Estimate, on a scale of 1-5 (inclusive), the time and effort required to review this PR by an experienced and knowledgeable developer. 1 means short and easy review , 5 means long and hard review. Take into account the size, complexity, quality, and the needed changes of the PR code diff.") {%- endif %} {%- if require_score %} score: str = Field(description="Rate this PR on a scale of 0-100 (inclusive), where 0 means the worst possible PR code, and 100 means PR code of the highest quality, without any bugs or performance issues, that is ready to be merged immediately and run in production at scale.") @@ -68,7 +68,7 @@ class Review(BaseModel): {%- if question_str %} insights_from_user_answers: str = Field(description="shortly summarize the insights you gained from the user's answers to the questions") {%- endif %} - possible_issues: str = Field(description="Does this PR code introduce clear issues, bugs, or major performance concerns? If there are no apparent issues, respond with 'No'. If there are any issues, describe them briefly. Use bullet points if more than one issue. Be specific, and provide examples if possible. Start each bullet point with a short specific header, such as: "- Possible Bug: ...", etc.") + key_issues_to_review: str = Field(description="Does this PR code introduce issues, bugs, or major performance concerns, which the PR reviewer should further investigate ? If there are no apparent issues, respond with 'No'. If there are any issues, describe them briefly. Use bullet points if more than one issue. Be specific, and provide examples if possible. Start each bullet point with a short specific header, such as: "- Possible Bug: ...", etc.") {%- if require_security_review %} security_concerns: str = Field(description="does this PR code introduce possible vulnerabilities such as exposure of sensitive information (e.g., API keys, secrets, passwords), or security concerns like SQL injection, XSS, CSRF, and others ? Answer 'No' if there are no possible issues. If there are security concerns or issues, start your answer with a short header, such as: 'Sensitive information exposure: ...', 'SQL injection: ...' etc. Explain your answer. Be specific and give examples if possible") {%- endif %} @@ -101,14 +101,14 @@ Example output: review: {%- if require_estimate_effort_to_review %} estimated_effort_to_review_[1-5]: | - 3, because ... + 3 {%- endif %} {%- if require_score %} score: 89 {%- endif %} relevant_tests: | No - possible_issues: | + key_issues_to_review: | No security_concerns: | No diff --git a/pr_agent/tools/pr_reviewer.py b/pr_agent/tools/pr_reviewer.py index baa0ccc4a..68267232e 100644 --- a/pr_agent/tools/pr_reviewer.py +++ b/pr_agent/tools/pr_reviewer.py @@ -140,7 +140,7 @@ async def run(self) -> None: if get_settings().pr_reviewer.persistent_comment and not self.incremental.is_incremental: final_update_message = get_settings().pr_reviewer.final_update_message self.git_provider.publish_persistent_comment(pr_review, - initial_header="## PR Review ๐Ÿ”", + initial_header="## PR Reviewer Guide ๐Ÿ”", update_header=True, final_update_message=final_update_message, ) else: @@ -193,10 +193,15 @@ def _prepare_pr_review(self) -> str: the feedback. """ data = load_yaml(self.prediction.strip(), - keys_fix_yaml=["estimated_effort_to_review_[1-5]:", "security_concerns:", "possible_issues:", + keys_fix_yaml=["estimated_effort_to_review_[1-5]:", "security_concerns:", "key_issues_to_review:", "relevant_file:", "relevant_line:", "suggestion:"]) github_action_output(data, 'review') + # move data['review'] 'key_issues_to_review' key to the end of the dictionary + if 'key_issues_to_review' in data['review']: + key_issues_to_review = data['review'].pop('key_issues_to_review') + data['review']['key_issues_to_review'] = key_issues_to_review + if 'code_feedback' in data: code_feedback = data['code_feedback'] @@ -260,7 +265,7 @@ def _publish_inline_code_comments(self) -> None: return data = load_yaml(self.prediction.strip(), - keys_fix_yaml=["estimated_effort_to_review_[1-5]:", "security_concerns:", "possible_issues:", + keys_fix_yaml=["estimated_effort_to_review_[1-5]:", "security_concerns:", "key_issues_to_review:", "relevant_file:", "relevant_line:", "suggestion:"]) comments: List[str] = [] for suggestion in data.get('code_feedback', []): diff --git a/tests/unittest/test_convert_to_markdown.py b/tests/unittest/test_convert_to_markdown.py index 676d0eea8..3b0ce3eac 100644 --- a/tests/unittest/test_convert_to_markdown.py +++ b/tests/unittest/test_convert_to_markdown.py @@ -52,7 +52,7 @@ def test_simple_dictionary_input(self): 'suggestion': "Consider raising an exception or logging a warning when 'pr_url' attribute is not found. This can help in debugging issues related to the absence of 'pr_url' in instances where it's expected. [important]\n", 'relevant_line': '[return ""](https://github.com/Codium-ai/pr-agent-pro/pull/102/files#diff-52d45f12b836f77ed1aef86e972e65404634ea4e2a6083fb71a9b0f9bb9e062fR199)'}]} - expected_output = '## PR Review ๐Ÿ”\n\n
{emoji} {key_nice}\n\n{value}\n\n
\n\n\n\n\n\n
โฑ๏ธ Estimated effort to review [1-5]\n\n1, because the changes are minimal and straightforward, focusing on a single functionality addition.\n\n\n
๐Ÿงช Relevant tests\n\nNo\n\n\n
โšก Possible issues\n\nNo\n\n
๐Ÿ”’ Security concerns\n\nNo\n\n
\n\n\n
Code feedback:\n\n
relevant filepr_agent/git_providers/git_provider.py\n
suggestion      \n\n\n\nConsider raising an exception or logging a warning when \'pr_url\' attribute is not found. This can help in debugging issues related to the absence of \'pr_url\' in instances where it\'s expected. [important]\n\n\n
relevant linereturn ""
\n\n
' + expected_output = '## PR Reviewer Guide ๐Ÿ”\n\n\n\n\n\n\n\n
โฑ๏ธ Estimated effort to review [1-5]\n\n1, because the changes are minimal and straightforward, focusing on a single functionality addition.\n\n\n
๐Ÿงช Relevant tests\n\nNo\n\n\n
โšก Possible issues\n\nNo\n\n
๐Ÿ”’ Security concerns\n\nNo\n\n
\n\n\n
Code feedback:\n\n
relevant filepr_agent/git_providers/git_provider.py\n
suggestion      \n\n\n\nConsider raising an exception or logging a warning when \'pr_url\' attribute is not found. This can help in debugging issues related to the absence of \'pr_url\' in instances where it\'s expected. [important]\n\n\n
relevant linereturn ""
\n\n
' assert convert_to_markdown(input_data).strip() == expected_output.strip() From e5f269040e3b8288297518b7776c39a08f8e37f9 Mon Sep 17 00:00:00 2001 From: mrT23 Date: Sun, 9 Jun 2024 14:32:50 +0300 Subject: [PATCH 2/4] Update PR review prompts and terminology for clarity and consistency --- tests/unittest/test_convert_to_markdown.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unittest/test_convert_to_markdown.py b/tests/unittest/test_convert_to_markdown.py index 3b0ce3eac..e4918bebf 100644 --- a/tests/unittest/test_convert_to_markdown.py +++ b/tests/unittest/test_convert_to_markdown.py @@ -52,7 +52,7 @@ def test_simple_dictionary_input(self): 'suggestion': "Consider raising an exception or logging a warning when 'pr_url' attribute is not found. This can help in debugging issues related to the absence of 'pr_url' in instances where it's expected. [important]\n", 'relevant_line': '[return ""](https://github.com/Codium-ai/pr-agent-pro/pull/102/files#diff-52d45f12b836f77ed1aef86e972e65404634ea4e2a6083fb71a9b0f9bb9e062fR199)'}]} - expected_output = '## PR Reviewer Guide ๐Ÿ”\n\n\n\n\n\n\n\n
โฑ๏ธ Estimated effort to review [1-5]\n\n1, because the changes are minimal and straightforward, focusing on a single functionality addition.\n\n\n
๐Ÿงช Relevant tests\n\nNo\n\n\n
โšก Possible issues\n\nNo\n\n
๐Ÿ”’ Security concerns\n\nNo\n\n
\n\n\n
Code feedback:\n\n
relevant filepr_agent/git_providers/git_provider.py\n
suggestion      \n\n\n\nConsider raising an exception or logging a warning when \'pr_url\' attribute is not found. This can help in debugging issues related to the absence of \'pr_url\' in instances where it\'s expected. [important]\n\n\n
relevant linereturn ""
\n\n
' + expected_output = '## PR Reviewer Guide ๐Ÿ”\n\n\n\n\n\n\n\n
โฑ๏ธ Estimated effort to review [1-5]\n\n1, because the changes are minimal and straightforward, focusing on a single functionality addition.\n\n\n
๐Ÿงช Relevant tests\n\nNo\n\n\n
โšก Possible issues\n\nNo\n\n\n
๐Ÿ”’ Security concerns\n\nNo\n\n
\n\n\n
Code feedback:\n\n
relevant filepr_agent/git_providers/git_provider.py\n
suggestion      \n\n\n\nConsider raising an exception or logging a warning when \'pr_url\' attribute is not found. This can help in debugging issues related to the absence of \'pr_url\' in instances where it\'s expected. [important]\n\n\n
relevant linereturn ""
\n\n
' assert convert_to_markdown(input_data).strip() == expected_output.strip() From 1106dccc4f86730b76cf195f652fdc73582de75e Mon Sep 17 00:00:00 2001 From: mrT23 Date: Sun, 9 Jun 2024 14:34:51 +0300 Subject: [PATCH 3/4] Update PR review prompts and terminology for clarity and consistency --- pr_agent/settings/pr_reviewer_prompts.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pr_agent/settings/pr_reviewer_prompts.toml b/pr_agent/settings/pr_reviewer_prompts.toml index 9f4d73f10..5c717b0fa 100644 --- a/pr_agent/settings/pr_reviewer_prompts.toml +++ b/pr_agent/settings/pr_reviewer_prompts.toml @@ -68,7 +68,7 @@ class Review(BaseModel): {%- if question_str %} insights_from_user_answers: str = Field(description="shortly summarize the insights you gained from the user's answers to the questions") {%- endif %} - key_issues_to_review: str = Field(description="Does this PR code introduce issues, bugs, or major performance concerns, which the PR reviewer should further investigate ? If there are no apparent issues, respond with 'No'. If there are any issues, describe them briefly. Use bullet points if more than one issue. Be specific, and provide examples if possible. Start each bullet point with a short specific header, such as: "- Possible Bug: ...", etc.") + key_issues_to_review: str = Field(description="Does this PR code introduce issues, bugs, or major performance concerns, which the PR reviewer should further investigate ? If there are no apparent issues, respond with 'None'. If there are any issues, describe them briefly. Use bullet points if more than one issue. Be specific, and provide examples if possible. Start each bullet point with a short specific header, such as: "- Possible Bug: ...", etc.") {%- if require_security_review %} security_concerns: str = Field(description="does this PR code introduce possible vulnerabilities such as exposure of sensitive information (e.g., API keys, secrets, passwords), or security concerns like SQL injection, XSS, CSRF, and others ? Answer 'No' if there are no possible issues. If there are security concerns or issues, start your answer with a short header, such as: 'Sensitive information exposure: ...', 'SQL injection: ...' etc. Explain your answer. Be specific and give examples if possible") {%- endif %} From 306fd3d064707a2b4eee78d8734a2d939306c636 Mon Sep 17 00:00:00 2001 From: mrT23 Date: Mon, 10 Jun 2024 08:13:48 +0300 Subject: [PATCH 4/4] Update PR review prompts and terminology for clarity and consistency --- docs/docs/tools/review.md | 51 +++++++++++++------- pr_agent/git_providers/bitbucket_provider.py | 2 +- pr_agent/git_providers/github_provider.py | 2 +- pr_agent/git_providers/gitlab_provider.py | 2 +- 4 files changed, 36 insertions(+), 21 deletions(-) diff --git a/docs/docs/tools/review.md b/docs/docs/tools/review.md index 3f69186eb..f0a3c057e 100644 --- a/docs/docs/tools/review.md +++ b/docs/docs/tools/review.md @@ -1,10 +1,13 @@ ## Overview -The `review` tool scans the PR code changes, and automatically generates a PR review. +The `review` tool scans the PR code changes, and generates a list of feedbacks about the PR, aiming to aid the reviewing process. +
The tool can be triggered automatically every time a new PR is [opened](../usage-guide/automations_and_usage.md#github-app-automatic-tools-when-a-new-pr-is-opened), or can be invoked manually by commenting on any PR: ``` /review ``` +Note that the main purpose of the `review` tool is to provide the **PR reviewer** with useful feedbacks and insights. The PR author, in contrast, may prefer to save time and focus on the output of the [improve](./improve.md) tool, which provides actionable code suggestions. + ## Example usage ### Manual triggering @@ -50,19 +53,27 @@ Note that the incremental mode is only available for GitHub. ![incremental review](https://codium.ai/images/pr_agent/incremental_review_2.png){width=512} -### PR Reflection +[//]: # (### PR Reflection) -By invoking: -``` -/reflect_and_review -``` -The tool will first ask the author questions about the PR, and will guide the review based on their answers. +[//]: # () +[//]: # (By invoking:) + +[//]: # (```) -![reflection questions](https://codium.ai/images/pr_agent/reflection_questions.png){width=512} +[//]: # (/reflect_and_review) -![reflection answers](https://codium.ai/images/pr_agent/reflection_answers.png){width=512} +[//]: # (```) -![reflection insights](https://codium.ai/images/pr_agent/reflection_insights.png){width=512} +[//]: # (The tool will first ask the author questions about the PR, and will guide the review based on their answers.) + +[//]: # () +[//]: # (![reflection questions](https://codium.ai/images/pr_agent/reflection_questions.png){width=512}) + +[//]: # () +[//]: # (![reflection answers](https://codium.ai/images/pr_agent/reflection_answers.png){width=512}) + +[//]: # () +[//]: # (![reflection insights](https://codium.ai/images/pr_agent/reflection_insights.png){width=512}) @@ -167,7 +178,7 @@ If enabled, the `review` tool can approve a PR when a specific comment, `/review !!! tip "General guidelines" - The `review` tool provides a collection of possible feedbacks about a PR. + The `review` tool provides a collection of configurable feedbacks about a PR. It is recommended to review the [Configuration options](#configuration-options) section, and choose the relevant options for your use case. Some of the features that are disabled by default are quite useful, and should be considered for enabling. For example: @@ -183,13 +194,6 @@ If enabled, the `review` tool can approve a PR when a specific comment, `/review Meaning the `review` tool will run automatically on every PR, without providing code suggestions. Edit this field to enable/disable the tool, or to change the used configurations. -!!! tip "Code suggestions" - - If you set `num_code_suggestions`>0 , the `review` tool will also provide code suggestions. - - Notice If you are interested **only** in the code suggestions, it is recommended to use the [`improve`](./improve.md) feature instead, since it is a dedicated only to code suggestions, and usually gives better results. - Use the `review` tool if you want to get more comprehensive feedback, which includes code suggestions as well. - !!! tip "Possible labels from the review tool" The `review` tool can auto-generate two specific types of labels for a PR: @@ -244,3 +248,14 @@ If enabled, the `review` tool can approve a PR when a specific comment, `/review [pr_reviewer] maximal_review_effort = 5 ``` + +[//]: # (!!! tip "Code suggestions") + +[//]: # () +[//]: # ( If you set `num_code_suggestions`>0 , the `review` tool will also provide code suggestions.) + +[//]: # ( ) +[//]: # ( Notice If you are interested **only** in the code suggestions, it is recommended to use the [`improve`](./improve.md) feature instead, since it is a dedicated only to code suggestions, and usually gives better results.) + +[//]: # ( Use the `review` tool if you want to get more comprehensive feedback, which includes code suggestions as well.) + diff --git a/pr_agent/git_providers/bitbucket_provider.py b/pr_agent/git_providers/bitbucket_provider.py index 3516516c0..27a88a767 100644 --- a/pr_agent/git_providers/bitbucket_provider.py +++ b/pr_agent/git_providers/bitbucket_provider.py @@ -172,7 +172,7 @@ def publish_persistent_comment(self, pr_comment: str, latest_commit_url = self.get_latest_commit_url() comment_url = self.get_comment_url(comment) if update_header: - updated_header = f"{initial_header}\n\n### ({name.capitalize()} updated until commit {latest_commit_url})\n" + updated_header = f"{initial_header}\n\n#### ({name.capitalize()} updated until commit {latest_commit_url})\n" pr_comment_updated = pr_comment.replace(initial_header, updated_header) else: pr_comment_updated = pr_comment diff --git a/pr_agent/git_providers/github_provider.py b/pr_agent/git_providers/github_provider.py index f2be54a3e..36f152018 100644 --- a/pr_agent/git_providers/github_provider.py +++ b/pr_agent/git_providers/github_provider.py @@ -217,7 +217,7 @@ def publish_persistent_comment(self, pr_comment: str, latest_commit_url = self.get_latest_commit_url() comment_url = self.get_comment_url(comment) if update_header: - updated_header = f"{initial_header}\n\n### ({name.capitalize()} updated until commit {latest_commit_url})\n" + updated_header = f"{initial_header}\n\n#### ({name.capitalize()} updated until commit {latest_commit_url})\n" pr_comment_updated = pr_comment.replace(initial_header, updated_header) else: pr_comment_updated = pr_comment diff --git a/pr_agent/git_providers/gitlab_provider.py b/pr_agent/git_providers/gitlab_provider.py index 3a2e1c850..ad18ba1df 100644 --- a/pr_agent/git_providers/gitlab_provider.py +++ b/pr_agent/git_providers/gitlab_provider.py @@ -160,7 +160,7 @@ def publish_persistent_comment(self, pr_comment: str, latest_commit_url = self.get_latest_commit_url() comment_url = self.get_comment_url(comment) if update_header: - updated_header = f"{initial_header}\n\n### ({name.capitalize()} updated until commit {latest_commit_url})\n" + updated_header = f"{initial_header}\n\n#### ({name.capitalize()} updated until commit {latest_commit_url})\n" pr_comment_updated = pr_comment.replace(initial_header, updated_header) else: pr_comment_updated = pr_comment