feat(INFRA-3081): add --require-pr-numbers flag for generation-time filtering (#253)

## What is the current state of things and why does it need to change?

Currently, `auto-changelog` includes all commits in the changelog, even
direct commits without PR numbers. For projects following strict
PR-based workflows (like MetaMask Extension), these direct commits
should be excluded to ensure all changelog entries represent reviewed
changes.

## What is the solution your changes offer and how does it work?

### 1. New `--requirePrNumbers` Flag
Adds an opt-in CLI flag that filters out commits without PR numbers at
generation time. Defaults to `false` for backward compatibility.

**Files changed:** `cli.ts`, `update-changelog.ts`, `get-new-changes.ts`

### 2. Graceful 404 Handling for Forked Repos
When running on forked repositories, the tool fetches PR details from
the fork. PRs originating from the upstream repo return 404 errors.
Added `try-catch` to return empty labels instead of crashing.

### 3. HTML Comment Stripping
PR templates often contain HTML comments (`<!-- ... -->`) that were
breaking markdown rendering in changelogs. Added `stripHtmlComments()`
function to clean descriptions.

### 4. `CHANGELOG entry: null` Fix
PRs marked with `CHANGELOG entry: null` were appearing as "Null (#PR)"
due to:
- Trailing whitespace not being trimmed
- Case-sensitive comparison (only matched lowercase `null`)

Fixed by adding `.trim()` and case-insensitive comparison at both check
points.

> **Note:** Fixes 2-4 were discovered during integration testing and are
prerequisites for the `--requirePrNumbers` flag to work correctly in
production environments with forked repos.

## Usage

```bash
yarn auto-changelog update --requirePrNumbers --autoCategorize --useChangelogEntry --useShortPrLink --rc
```

## Testing

Tested with
[consensys-test/metamask-extension-test](https://github.com/consensys-test/metamask-extension-test)
on `release/1100.0.0`:

| Test | Result |
|------|--------|
| PR Commit `feat: test commit with PR for 1100.0.0 (#40)` | ✅
**INCLUDED** |
| Direct Commit `chore: direct commit without PR for 1100.0.0` | ✅
**EXCLUDED** |
| `Null (#PR)` entries | ✅ **NONE** (case-insensitive fix working) |
| [Generated
Changelog](https://github.com/consensys-test/metamask-extension-test/blob/release/1100.0.0-Changelog/CHANGELOG.md)
| ✅ Correct |

### Changelog Excerpt

```markdown
## [1100.0.0]

### Added
- feat: test commit with PR for 1100.0.0 (#40)
- Design upgrades for Confirmations (#38399)
- Fix amount component to be only accept numeric inputs (#38235)
...
```

**Note:** `N/A` and `None` entries still appear (e.g., `N/A (#37810)`)
because these are different strings from `null` - they're valid
descriptions that PRs chose to use.

## References

- Complements duplicate detection work: #251

Author: Qbandev

README.md

@@ -38,6 +38,14 @@ or
 
 `yarn run auto-changelog update --useShortPrLink`
 
+#### Require PR numbers (filter out commits without PR numbers)
+
+- Only include commits that have associated PR numbers in the changelog
+- Commits without PR numbers (like direct commits) will be filtered out
+- This is useful for projects that want to ensure all changelog entries come from reviewed pull requests
+
+`yarn run auto-changelog update --requirePrNumbers`
+
 #### Update the current release section of the changelog
 
 `yarn run auto-changelog update --rc`
@@ -50,6 +58,10 @@ or
 
 `yarn run auto-changelog update --autoCategorize --useChangelogEntry --useShortPrLink --rc`
 
+### With requirePrNumbers (for stricter PR-based workflows)
+
+`yarn run auto-changelog update --autoCategorize --useChangelogEntry --useShortPrLink --requirePrNumbers --rc`
+
 #### Update the changelog for a renamed package
 
 This option is designed to be used for packages that live in a monorepo.
@@ -132,6 +144,7 @@ const updatedChangelog = await updateChangelog({
   currentVersion: '1.0.0',
   repoUrl: 'https://github.com/ExampleUsernameOrOrganization/ExampleRepository',
   isReleaseCandidate: false,
+  requirePrNumbers: false, // Optional: set to true to filter out commits without PR numbers
 });
 await fs.writeFile('CHANGELOG.md', updatedChangelog);
 ```

src/cli.ts

@@ -102,6 +102,10 @@ type UpdateOptions = {
    * Whether to use short PR links in the changelog entries
    */
   useShortPrLink: boolean;
+  /**
+   * Whether to require PR numbers for all commits. If true, commits without PR numbers are filtered out.
+   */
+  requirePrNumbers: boolean;
 };
 
 /**
@@ -121,6 +125,7 @@ type UpdateOptions = {
  * @param options.autoCategorize - Whether to categorize commits automatically based on their messages.
  * @param options.useChangelogEntry - Whether to read `CHANGELOG entry:` from the commit body and the no-changelog label.
  * @param options.useShortPrLink - Whether to use short PR links in the changelog entries.
+ * @param options.requirePrNumbers - Whether to require PR numbers for all commits. If true, commits without PR numbers are filtered out.
  */
 async function update({
   changelogPath,
@@ -134,6 +139,7 @@ async function update({
   autoCategorize,
   useChangelogEntry,
   useShortPrLink,
+  requirePrNumbers,
 }: UpdateOptions) {
   const changelogContent = await readChangelog(changelogPath);
 
@@ -149,6 +155,7 @@ async function update({
     autoCategorize,
     useChangelogEntry,
     useShortPrLink,
+    requirePrNumbers,
   });
 
   if (newChangelogContent) {
@@ -357,6 +364,12 @@ async function main() {
             description: 'Use short PR links in the changelog entries',
             type: 'boolean',
           })
+          .option('requirePrNumbers', {
+            default: false,
+            description:
+              'Only include commits with PR numbers in the changelog. Commits without PR numbers will be filtered out',
+            type: 'boolean',
+          })
           .epilog(updateEpilog),
     )
     .command(
@@ -416,6 +429,7 @@ async function main() {
     prLinks,
     useChangelogEntry,
     useShortPrLink,
+    requirePrNumbers,
   } = argv;
   let { currentVersion } = argv;
 
@@ -547,6 +561,7 @@ async function main() {
       autoCategorize,
       useChangelogEntry: Boolean(useChangelogEntry),
       useShortPrLink: Boolean(useShortPrLink),
+      requirePrNumbers: Boolean(requirePrNumbers),
     });
   } else if (command === 'validate') {
     let packageRename: PackageRename | undefined;

src/get-new-changes.ts

@@ -14,6 +14,7 @@ export type AddNewCommitsOptions = {
   projectRootDirectory?: string;
   useChangelogEntry: boolean;
   useShortPrLink: boolean;
+  requirePrNumbers?: boolean;
 };
 
 // Get array of all ConventionalCommitType values
@@ -61,6 +62,22 @@ function removeConventionalCommitPrefixIfPresent(message: string) {
   return message.replace(regex, '');
 }
 
+/**
+ * Remove HTML comments from the given message.
+ * Handles both complete comments (<!-- ... -->) and unclosed comments (<!-- ...).
+ * This prevents PR template content from breaking the changelog markdown.
+ *
+ * @param message - The changelog entry message.
+ * @returns The message without HTML comments.
+ */
+function stripHtmlComments(message: string): string {
+  // Remove complete HTML comments (<!-- ... -->)
+  let result = message.replace(/<!--[\s\S]*?-->/gu, '');
+  // Remove unclosed HTML comments (<!-- without closing -->)
+  result = result.replace(/<!--[\s\S]*$/gu, '');
+  return result.trim();
+}
+
 type Commit = {
   prNumber?: string;
   subject: string;
@@ -122,11 +139,12 @@ async function getCommits(
         const changelogMatch = body.match(/\nCHANGELOG entry:\s(\S.+?)\n\n/su);
 
         if (changelogMatch) {
-          const changelogEntry = changelogMatch[1].replace('\n', ' ');
+          const changelogEntry = changelogMatch[1].replace('\n', ' ').trim();
 
           description = changelogEntry; // This may be string 'null' to indicate no description
 
-          if (description !== 'null') {
+          // Check for 'null' (case-insensitive) to exclude entries marked as no-changelog
+          if (description.toLowerCase() !== 'null') {
             // Remove outer backticks if present. Example: `feat: new feature description` -> feat: new feature description
             description = removeOuterBackticksIfPresent(description);
 
@@ -141,7 +159,8 @@ async function getCommits(
           description = subject.match(/^(.+)\s\(#\d+\)/u)?.[1] ?? '';
         }
 
-        if (description !== 'null') {
+        // Filter out entries marked as no-changelog (case-insensitive null check)
+        if (description.toLowerCase() !== 'null') {
           const prLabels = await getPrLabels(repoUrl, prNumber);
 
           if (prLabels.includes('no-changelog')) {
@@ -233,6 +252,7 @@ function deduplicateCommits(
  * current git repository.
  * @param options.useChangelogEntry - Whether to use `CHANGELOG entry:` from the commit body and the no-changelog label.
  * @param options.useShortPrLink - Whether to use short PR links in the changelog entries.
+ * @param options.requirePrNumbers - Whether to require PR numbers for all commits. If true, commits without PR numbers are filtered out.
  * @returns A list of new change entries to add to the changelog, based on commits made since the last release.
  */
 export async function getNewChangeEntries({
@@ -243,6 +263,7 @@ export async function getNewChangeEntries({
   projectRootDirectory,
   useChangelogEntry,
   useShortPrLink,
+  requirePrNumbers = false,
 }: AddNewCommitsOptions) {
   const commitRange =
     mostRecentTag === null ? 'HEAD' : `${mostRecentTag}..HEAD`;
@@ -256,8 +277,12 @@ export async function getNewChangeEntries({
     useChangelogEntry,
   );
 
+  const filteredPrCommits = requirePrNumbers
+    ? commits.filter((commit) => commit.prNumber !== undefined)
+    : commits;
+
   const newCommits = deduplicateCommits(
-    commits,
+    filteredPrCommits,
     loggedPrNumbers,
     loggedDescriptions,
   );
@@ -266,6 +291,11 @@ export async function getNewChangeEntries({
     // Handle edge case where PR description includes multiple CHANGELOG entries
     let newDescription = description?.replace(/CHANGELOG entry: /gu, '');
 
+    // Strip HTML comments that may come from PR templates to prevent broken markdown
+    if (newDescription) {
+      newDescription = stripHtmlComments(newDescription);
+    }
+
     // For merge commits, use the description for categorization because the subject
     // is "Merge pull request #123..." which would be incorrectly excluded
     const subjectForCategorization = isMergeCommit ? description : subject;
@@ -323,16 +353,31 @@ async function getPrLabels(
 
   const { owner, repo } = getOwnerAndRepoFromUrl(repoUrl);
 
-  const { data: pullRequest } = await github.rest.pulls.get({
-    owner,
-    repo,
-    // eslint-disable-next-line @typescript-eslint/naming-convention
-    pull_number: Number(prNumber),
-  });
-
-  if (pullRequest) {
-    const labels = pullRequest.labels.map((label: any) => label.name);
-    return labels;
+  try {
+    const { data: pullRequest } = await github.rest.pulls.get({
+      owner,
+      repo,
+      // eslint-disable-next-line @typescript-eslint/naming-convention
+      pull_number: Number(prNumber),
+    });
+
+    if (pullRequest) {
+      const labels = pullRequest.labels.map((label: any) => label.name);
+      return labels;
+    }
+  } catch (error: unknown) {
+    // If PR doesn't exist (404), return empty labels instead of throwing
+    if (
+      error instanceof Error &&
+      'status' in error &&
+      (error as { status: number }).status === 404
+    ) {
+      console.warn(
+        `PR #${prNumber} not found in ${owner}/${repo}, skipping label check`,
+      );
+      return [];
+    }
+    throw error;
   }
 
   return [];

src/update-changelog.ts

@@ -112,6 +112,10 @@ export type UpdateChangelogOptions = {
    * Whether to use short PR links in the changelog entries.
    */
   useShortPrLink?: boolean;
+  /**
+   * Whether to require PR numbers for all commits. If true, commits without PR numbers are filtered out.
+   */
+  requirePrNumbers?: boolean;
 };
 
 /**
@@ -140,6 +144,7 @@ export type UpdateChangelogOptions = {
  * based on commit message prefixes.
  * @param options.useChangelogEntry - Whether to use `CHANGELOG entry:` from the commit body and the no-changelog label.
  * @param options.useShortPrLink - Whether to use short PR links in the changelog.
+ * @param options.requirePrNumbers - Whether to require PR numbers for all commits. If true, commits without PR numbers are filtered out.
  * @returns The updated changelog text.
  */
 export async function updateChangelog({
@@ -154,6 +159,7 @@ export async function updateChangelog({
   autoCategorize,
   useChangelogEntry = false,
   useShortPrLink = false,
+  requirePrNumbers = false,
 }: UpdateChangelogOptions): Promise<string | undefined> {
   const changelog = parseChangelog({
     changelogContent,
@@ -204,6 +210,7 @@ export async function updateChangelog({
     projectRootDirectory,
     useChangelogEntry,
     useShortPrLink,
+    requirePrNumbers,
   });
 
   for (const entry of newChangeEntries.reverse()) {