fix(INFRA-3081): implement duplicate detection for all commit types (#254)

## Summary

Implements duplicate detection for changelog entries to prevent
re-adding the same commits across multiple workflow runs.

## Problem

The auto-changelog tool was creating duplicate entries when CI ran
multiple times on a release branch. The existing deduplication only
tracked PR numbers, so direct commits (without PR numbers) like "Update
Attributions" or "chore: bump version" would be added repeatedly.

## Solution

### 1. Dual-Tracking Duplicate Detection
- **PR-based commits**: Check if PR number already exists in changelog
- **Direct commits**: Check if exact description text already exists in
changelog

### 2. Handle Squash Merge Duplicates
Squash merges create two commits with the same content:
- Feature branch commit: `feat: Add feature` (no PR number)
- Merge commit: `feat: Add feature (#123)` (has PR number)

The feature branch commit is now skipped when its corresponding merge
commit exists.

### 3. Fix Merge Commit Categorization
Regular merge commits (format: `Merge pull request #123 from branch`)
were being excluded because categorization used the merge subject
instead of the actual PR content. Now uses the extracted description
from the commit body for proper categorization.

## Changes

- Added `loggedDescriptions` parameter to track all changelog entry
descriptions
- Updated duplicate detection to check both PR numbers and descriptions
- Pre-build Set of PR descriptions for O(1) lookup performance
- Refined exclusion keyword `'merge'` → `'Merge pull request'` to avoid
false exclusions
- Use description for categorizing merge commits instead of subject

## Testing

Verified with test repository
[`consensys-test/metamask-extension-test`](https://github.com/consensys-test/metamask-extension-test)
using multiple release branches with:
- Regular merge PRs
- Squash merge PRs  
- Direct commits (feat, fix, docs, chore)
- Multiple workflow runs to test idempotency

**Results:** ✅ No duplicates, all commits correctly categorized, merge
commits properly handled
- [Latest job
run](https://github.com/consensys-test/metamask-extension-test/actions/runs/19496632321/job/55800299530)
-
[Changelog](https://github.com/consensys-test/metamask-extension-test/blob/release/13.10.0-Changelog/CHANGELOG.md)

## Related

Fixes INFRA-3081


<!-- CURSOR_SUMMARY -->
---

> [!NOTE]
> Add robust duplicate detection using PR numbers and exact descriptions
(incl. squash/merge cases), update exclusions, and add comprehensive
tests.
> 
> - **Changelog generation (`src/get-new-changes.ts`)**:
> - Add `loggedDescriptions` support and dedup logic for PR-based and
direct commits, skipping direct commits when a matching PR merge exists.
> - Use description (not subject) for merge-commit categorization;
append PR link with optional short format.
> - Replace assertion with explicit error when `git show` subject is
empty; refine GITHUB_TOKEN retrieval.
> - **Changelog update (`src/update-changelog.ts`)**:
> - Extract trimmed `loggedDescriptions` from existing changelog and
pass to `getNewChangeEntries`.
> - **Exclusions (`src/constants.ts`)**:
>   - Adjust exclusion keyword from `merge` to `Merge pull request`.
> - **Tests**:
> - Add `src/get-new-changes.test.ts` with cases for PR vs direct
commits, duplicate detection across runs, squash/merge handling, link
formatting, and edge cases.
> 
> <sup>Written by [Cursor
Bugbot](https://cursor.com/dashboard?tab=bugbot) for commit
d4fea63. Configure
[here](https://cursor.com/dashboard?tab=bugbot).</sup>
<!-- /CURSOR_SUMMARY -->

---------

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

Author: Qbandev

src/constants.ts

@@ -103,6 +103,6 @@ export const keywordsToIndicateExcluded: string[] = [
   'e2e',
   'flaky test',
   'INFRA-',
-  'merge',
+  'Merge pull request',
   'New Crowdin translations',
 ].map((word) => word.toLowerCase());

src/get-new-changes.test.ts

@@ -0,0 +1,339 @@
+import { getNewChangeEntries } from './get-new-changes';
+import { runCommand, runCommandAndSplit } from './run-command';
+
+jest.mock('./run-command');
+
+const mockRunCommand = runCommand as jest.MockedFunction<typeof runCommand>;
+const mockRunCommandAndSplit = runCommandAndSplit as jest.MockedFunction<
+  typeof runCommandAndSplit
+>;
+
+const repoUrl = 'https://github.com/MetaMask/metamask-mobile';
+
+describe('getNewChangeEntries', () => {
+  beforeEach(() => {
+    mockRunCommandAndSplit.mockResolvedValue([]);
+  });
+
+  describe('PR-tagged commits', () => {
+    it('should include commits with PR numbers', async () => {
+      mockRunCommandAndSplit.mockResolvedValueOnce(['commit1', 'commit2']);
+      mockRunCommand
+        .mockResolvedValueOnce('add feature (#12345)')
+        .mockResolvedValueOnce('bug fix (#12346)');
+
+      const result = await getNewChangeEntries({
+        mostRecentTag: 'v1.0.0',
+        repoUrl,
+        loggedPrNumbers: [],
+        loggedDescriptions: [],
+        useChangelogEntry: false,
+        useShortPrLink: false,
+      });
+
+      expect(result).toStrictEqual([
+        {
+          description:
+            'add feature ([#12345](https://github.com/MetaMask/metamask-mobile/pull/12345))',
+          subject: 'add feature (#12345)',
+        },
+        {
+          description:
+            'bug fix ([#12346](https://github.com/MetaMask/metamask-mobile/pull/12346))',
+          subject: 'bug fix (#12346)',
+        },
+      ]);
+    });
+
+    it('should exclude commits with PR numbers already in changelog', async () => {
+      mockRunCommandAndSplit.mockResolvedValueOnce(['commit1', 'commit2']);
+      mockRunCommand
+        .mockResolvedValueOnce('add feature (#12345)')
+        .mockResolvedValueOnce('bug fix (#12346)');
+
+      const result = await getNewChangeEntries({
+        mostRecentTag: 'v1.0.0',
+        repoUrl,
+        loggedPrNumbers: ['12345'],
+        loggedDescriptions: [],
+        useChangelogEntry: false,
+        useShortPrLink: false,
+      });
+
+      expect(result).toStrictEqual([
+        {
+          description:
+            'bug fix ([#12346](https://github.com/MetaMask/metamask-mobile/pull/12346))',
+          subject: 'bug fix (#12346)',
+        },
+      ]);
+    });
+  });
+
+  describe('direct commits (no PR numbers)', () => {
+    it('should include direct commits', async () => {
+      mockRunCommandAndSplit.mockResolvedValueOnce(['commit1', 'commit2']);
+      mockRunCommand
+        .mockResolvedValueOnce('Update Attributions')
+        .mockResolvedValueOnce('Bump version to 7.58.0');
+
+      const result = await getNewChangeEntries({
+        mostRecentTag: 'v1.0.0',
+        repoUrl,
+        loggedPrNumbers: [],
+        loggedDescriptions: [],
+        useChangelogEntry: false,
+        useShortPrLink: false,
+      });
+
+      expect(result).toStrictEqual([
+        {
+          description: 'Update Attributions',
+          subject: 'Update Attributions',
+        },
+        {
+          description: 'Bump version to 7.58.0',
+          subject: 'Bump version to 7.58.0',
+        },
+      ]);
+    });
+
+    it('should exclude direct commits already in changelog', async () => {
+      mockRunCommandAndSplit.mockResolvedValueOnce(['commit1', 'commit2']);
+      mockRunCommand
+        .mockResolvedValueOnce('Update Attributions')
+        .mockResolvedValueOnce('Bump version to 7.58.0');
+
+      const result = await getNewChangeEntries({
+        mostRecentTag: 'v1.0.0',
+        repoUrl,
+        loggedPrNumbers: [],
+        loggedDescriptions: ['Update Attributions'],
+        useChangelogEntry: false,
+        useShortPrLink: false,
+      });
+
+      expect(result).toStrictEqual([
+        {
+          description: 'Bump version to 7.58.0',
+          subject: 'Bump version to 7.58.0',
+        },
+      ]);
+    });
+  });
+
+  describe('merge commits', () => {
+    it('should extract PR numbers from merge commits', async () => {
+      mockRunCommandAndSplit.mockResolvedValueOnce(['commit1', 'commit2']);
+      mockRunCommand
+        .mockResolvedValueOnce('Merge pull request #12345 from feature-branch')
+        .mockResolvedValueOnce('Merge pull request #12346 from fix-branch');
+      // Mock body fetches for merge commits
+      mockRunCommandAndSplit
+        .mockResolvedValueOnce(['implement new feature'])
+        .mockResolvedValueOnce(['fix critical bug']);
+
+      const result = await getNewChangeEntries({
+        mostRecentTag: 'v1.0.0',
+        repoUrl,
+        loggedPrNumbers: [],
+        loggedDescriptions: [],
+        useChangelogEntry: false,
+        useShortPrLink: false,
+      });
+
+      expect(result).toStrictEqual([
+        {
+          description:
+            'implement new feature ([#12345](https://github.com/MetaMask/metamask-mobile/pull/12345))',
+          subject: 'implement new feature',
+        },
+        {
+          description:
+            'fix critical bug ([#12346](https://github.com/MetaMask/metamask-mobile/pull/12346))',
+          subject: 'fix critical bug',
+        },
+      ]);
+    });
+  });
+
+  describe('squash merge deduplication', () => {
+    it('should skip direct commit when PR-tagged commit with same description exists', async () => {
+      // Simulates squash merge where both original and merged commits appear
+      mockRunCommandAndSplit.mockResolvedValueOnce(['commit1', 'commit2']);
+      mockRunCommand
+        .mockResolvedValueOnce('add new feature') // Direct commit (no PR)
+        .mockResolvedValueOnce('add new feature (#12345)'); // PR-tagged commit with same description
+
+      const result = await getNewChangeEntries({
+        mostRecentTag: 'v1.0.0',
+        repoUrl,
+        loggedPrNumbers: [],
+        loggedDescriptions: [],
+        useChangelogEntry: false,
+        useShortPrLink: false,
+      });
+
+      // Should only include the PR-tagged version
+      expect(result).toStrictEqual([
+        {
+          description:
+            'add new feature ([#12345](https://github.com/MetaMask/metamask-mobile/pull/12345))',
+          subject: 'add new feature (#12345)',
+        },
+      ]);
+    });
+  });
+
+  describe('duplicate detection', () => {
+    it('should return empty array when all commits are duplicates', async () => {
+      mockRunCommandAndSplit.mockResolvedValueOnce([
+        'commit1',
+        'commit2',
+        'commit3',
+      ]);
+      mockRunCommand
+        .mockResolvedValueOnce('add feature (#12345)')
+        .mockResolvedValueOnce('Update Attributions')
+        .mockResolvedValueOnce('Bump version');
+
+      const result = await getNewChangeEntries({
+        mostRecentTag: 'v1.0.0',
+        repoUrl,
+        loggedPrNumbers: ['12345'],
+        loggedDescriptions: ['Update Attributions', 'Bump version'],
+        useChangelogEntry: false,
+        useShortPrLink: false,
+      });
+
+      expect(result).toStrictEqual([]);
+    });
+  });
+
+  describe('edge cases', () => {
+    it('should return empty array when there are no commits', async () => {
+      mockRunCommandAndSplit.mockResolvedValueOnce([]);
+
+      const result = await getNewChangeEntries({
+        mostRecentTag: 'v1.0.0',
+        repoUrl,
+        loggedPrNumbers: [],
+        loggedDescriptions: [],
+        useChangelogEntry: false,
+        useShortPrLink: false,
+      });
+
+      expect(result).toStrictEqual([]);
+    });
+
+    it('should use HEAD as commit range when no tag is available', async () => {
+      mockRunCommandAndSplit.mockResolvedValueOnce(['commit1']);
+      mockRunCommand.mockResolvedValueOnce('add feature (#12345)');
+
+      const result = await getNewChangeEntries({
+        mostRecentTag: null,
+        repoUrl,
+        loggedPrNumbers: [],
+        loggedDescriptions: [],
+        useChangelogEntry: false,
+        useShortPrLink: false,
+      });
+
+      expect(mockRunCommandAndSplit).toHaveBeenCalledWith('git', [
+        'rev-list',
+        'HEAD',
+      ]);
+      expect(result).toStrictEqual([
+        {
+          description:
+            'add feature ([#12345](https://github.com/MetaMask/metamask-mobile/pull/12345))',
+          subject: 'add feature (#12345)',
+        },
+      ]);
+    });
+
+    it('should throw error when git show returns empty subject', async () => {
+      mockRunCommandAndSplit.mockResolvedValueOnce(['commit1']);
+      mockRunCommand.mockResolvedValueOnce('');
+
+      await expect(
+        getNewChangeEntries({
+          mostRecentTag: 'v1.0.0',
+          repoUrl,
+          loggedPrNumbers: [],
+          loggedDescriptions: [],
+          useChangelogEntry: false,
+          useShortPrLink: false,
+        }),
+      ).rejects.toThrow(
+        '"git show" returned empty subject for commit "commit1".',
+      );
+    });
+  });
+
+  describe('PR link formatting', () => {
+    it('should include full PR link when useShortPrLink is false', async () => {
+      mockRunCommandAndSplit.mockResolvedValueOnce(['commit1']);
+      mockRunCommand.mockResolvedValueOnce('add feature (#12345)');
+
+      const result = await getNewChangeEntries({
+        mostRecentTag: 'v1.0.0',
+        repoUrl,
+        loggedPrNumbers: [],
+        loggedDescriptions: [],
+        useChangelogEntry: false,
+        useShortPrLink: false,
+      });
+
+      expect(result).toStrictEqual([
+        {
+          description:
+            'add feature ([#12345](https://github.com/MetaMask/metamask-mobile/pull/12345))',
+          subject: 'add feature (#12345)',
+        },
+      ]);
+    });
+
+    it('should use short PR link when useShortPrLink is true', async () => {
+      mockRunCommandAndSplit.mockResolvedValueOnce(['commit1']);
+      mockRunCommand.mockResolvedValueOnce('add feature (#12345)');
+
+      const result = await getNewChangeEntries({
+        mostRecentTag: 'v1.0.0',
+        repoUrl,
+        loggedPrNumbers: [],
+        loggedDescriptions: [],
+        useChangelogEntry: false,
+        useShortPrLink: true,
+      });
+
+      expect(result).toStrictEqual([
+        {
+          description: 'add feature (#12345)',
+          subject: 'add feature (#12345)',
+        },
+      ]);
+    });
+
+    it('should not add PR link suffix for direct commits', async () => {
+      mockRunCommandAndSplit.mockResolvedValueOnce(['commit1']);
+      mockRunCommand.mockResolvedValueOnce('Update Attributions');
+
+      const result = await getNewChangeEntries({
+        mostRecentTag: 'v1.0.0',
+        repoUrl,
+        loggedPrNumbers: [],
+        loggedDescriptions: [],
+        useChangelogEntry: false,
+        useShortPrLink: false,
+      });
+
+      expect(result).toStrictEqual([
+        {
+          description: 'Update Attributions',
+          subject: 'Update Attributions',
+        },
+      ]);
+    });
+  });
+});