Add: script to automatically produce changelogs based on the commits

This needs post-processing before it becomes a good changelog,
but at least this helps in ordering and pruning what is possible
in an automated way.

.github/workflows/testing.yml

@@ -36,7 +36,7 @@ jobs:
     - name: Flake8
       uses: TrueBrain/actions-flake8@v2
       with:
-        path: backport
+        path: backport changelog
 
   black:
     name: Black
@@ -52,7 +52,7 @@ jobs:
       run: |
         python -m pip install --upgrade pip
         pip install black
-        black -l 120 --check backport
+        black -l 120 --check backport changelog
 
   check_annotations:
     name: Check Annotations

changelog/changelog.py

@@ -0,0 +1,279 @@
+"""
+Put this file in a master checkout under .github/.
+
+This assumes your git "origin" points to your fork, and "upstream" to upstream.
+This script will overwrite the branch "changelog".
+
+The GITHUB_TOKEN is a classic token you need to generate that has public_repo
+scope enabled. This token is used to fetch information from the GitHub API.
+
+Execute with:
+
+$ export GITHUB_TOKEN=ghp_XXX
+$ python3 .github/backport.py <last-commit-of-previous-release>
+"""
+
+import json
+import os
+import subprocess
+import sys
+
+BEARER_TOKEN = os.getenv("GITHUB_TOKEN")
+
+if not BEARER_TOKEN:
+    print("Please set the GITHUB_TOKEN environment variable.")
+    sys.exit(1)
+
+PRIORITY = {
+    "Feature": 1,
+    "Add": 2,
+    "Change": 3,
+    "Fix": 4,
+    "Remove": 5,
+}
+
+commit_pr_query = """
+query ($hash: String) {
+  repository(owner: "OpenTTD", name: "OpenTTD") {
+    ref(qualifiedName: "master") {
+      target {
+        ... on Commit {
+          history(first: 100, after: $hash) {
+            pageInfo{
+              endCursor
+            }
+            edges {
+              node {
+                oid
+                associatedPullRequests(first: 1) {
+                  edges {
+                    node {
+                      number
+                      labels(first: 10) {
+                        edges {
+                          node {
+                            name
+                          }
+                        }
+                      }
+                      closingIssuesReferences(first: 10) {
+                        edges {
+                          node {
+                            number
+                          }
+                        }
+                      }
+                    }
+                  }
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+}
+
+"""
+
+
+def do_query(query, variables):
+    query = query.replace("\n", "").replace("\\", "\\\\").replace('"', '\\"')
+    variables = json.dumps(variables).replace("\\", "\\\\").replace('"', '\\"')
+    res = subprocess.run(
+        [
+            "curl",
+            "--fail",
+            "-H",
+            f"Authorization: bearer {BEARER_TOKEN}",
+            "-X",
+            "POST",
+            "-d",
+            f'{{"query": "{query}", "variables": "{variables}"}}',
+            "https://api.github.com/graphql",
+        ],
+        capture_output=True,
+    )
+    if res.returncode != 0:
+        return None
+    return json.loads(res.stdout)
+
+
+def do_command(command):
+    return subprocess.run(command, capture_output=True)
+
+
+def main():
+    last_commit = sys.argv[1]
+
+    do_command(["git", "fetch", "upstream"])
+    do_command(["git", "checkout", f"upstream/master", "-B", "changelog"])
+
+    commit_list = do_command(["git", "log", "--pretty=format:%ce|%H|%s", f"{last_commit}..HEAD"])
+    commits = commit_list.stdout.decode().splitlines()
+
+    commit_to_pr = {}
+    backported = set()
+    commits_seen = set()
+    issues = {}
+
+    cache_filename = f".changelog-cache-{last_commit}.{len(commits)}"
+
+    if os.path.exists(cache_filename):
+        with open(cache_filename, "r") as f:
+            data = json.loads(f.read())
+            commit_to_pr = data["commit_to_pr"]
+            backported = set(data["backported"])
+            commits_seen = set(data["commits_seen"])
+            issues = data["issues"]
+    else:
+        print("Fetching commits and their associated PRs ... this might take a while ...")
+
+        count = len(commits)
+        variables = {}
+        while count > 0:
+            print(f"{count} commits left ...")
+
+            res = do_query(commit_pr_query, variables)
+            variables["hash"] = res["data"]["repository"]["ref"]["target"]["history"]["pageInfo"]["endCursor"]
+
+            # Walk all commits.
+            for edge in res["data"]["repository"]["ref"]["target"]["history"]["edges"]:
+                count -= 1
+                if count == 0:
+                    break
+                if not edge["node"]["associatedPullRequests"]["edges"]:
+                    continue
+
+                pr = edge["node"]["associatedPullRequests"]["edges"][0]["node"]
+
+                # Links the commit to a PR.
+                commit_to_pr[edge["node"]["oid"]] = pr["number"]
+                # Link the hashes between 6 and 10 in size to the list of "seen commits".
+                for i in range(6, 10):
+                    commits_seen.add(edge["node"]["oid"][0:i])
+
+                # Check if this PR was backported.
+                for label in pr["labels"]["edges"]:
+                    if label["node"]["name"] == "backported":
+                        backported.add(pr["number"])
+
+                # Track which issues were closed because of this PR.
+                if pr["closingIssuesReferences"]["edges"]:
+                    issues[pr["number"]] = []
+                    for issue in pr["closingIssuesReferences"]["edges"]:
+                        issues[pr["number"]].append(issue["node"]["number"])
+
+        with open(cache_filename, "w") as f:
+            f.write(
+                json.dumps(
+                    {
+                        "commit_to_pr": commit_to_pr,
+                        "backported": list(backported),
+                        "commits_seen": list(commits_seen),
+                        "issues": issues,
+                    }
+                )
+            )
+
+    messages = []
+
+    for commit in commit_list.stdout.decode().splitlines():
+        (email, hash, message) = commit.split("|", 2)
+
+        if email in ("translators@openttd.org",):
+            continue
+
+        # Ignore all commit messages that don't change functionality.
+        if message.startswith(("Codechange", "Codefix", "Doc", "Update", "Upgrade", "Cleanup", "Prepare", "Revert")):
+            continue
+        # Ignore everything related to the CI.
+        if "[CI]" in message or "[Dependabot]" in message or "[DorpsGek]" in message:
+            continue
+
+        pr = commit_to_pr.get(hash)
+        if pr is None:
+            pr = -1
+
+        # Skip everything already backported.
+        if pr in backported:
+            continue
+
+        # Remove the PR number from the commit message.
+        if message.endswith(f"(#{pr})"):
+            message = message[: -len(f"(#{pr})")]
+
+        commit_type, commit_message = message.strip().split(":", 1)
+        subject = None
+
+        # Remove trailing dots.
+        commit_message = commit_message.strip().rstrip(".")
+        commit_message = commit_message[0].upper() + commit_message[1:]
+        # If the string starts with [, capitalize the first letter after the ].
+        if commit_message.startswith("["):
+            part1, part2 = commit_message.split("]", 1)
+            part2 = part2.strip()
+            commit_message = part1 + "] " + part2[0].upper() + part2[1:]
+
+        if " " in commit_type:
+            commit_type, subject = commit_type.split(" ", 1)
+
+            if commit_type != "Fix":
+                # Remove subject if not a fix.
+                subject = None
+            else:
+                reference = False
+                ticket = None
+
+                # Check all parts of the subject for either a ticket or hash.
+                for sub in subject.split(","):
+                    sub = sub.strip()
+
+                    # If we reference a ticket, that will be the subject.
+                    if sub.startswith("#"):
+                        ticket = sub
+                        continue
+
+                    # If the hash is in our set of commits, it is a fix for
+                    # something unreleased; so don't mention it.
+                    if sub in commits_seen:
+                        reference = True
+                        break
+
+                if reference:
+                    continue
+                subject = ticket
+
+        if commit_type == "Fix" and issues.get(str(pr)):
+            issue_list = issues[str(pr)]
+
+            # Check if any of the linked issues are mentioned in the commit.
+            for issue in issue_list:
+                if subject and f"#{issue}" in subject:
+                    break
+            else:
+                # The linked issue is not mentioned. Create the link.
+                issue = ", ".join([f"#{issue}" for issue in issue_list])
+
+                if subject and subject != issue:
+                    print(
+                        f"WARNING: commit {hash} has a different references than the PR {pr}: '{subject}' vs '{issue}'"
+                    )
+                subject = issue
+
+        message = commit_type
+        if subject:
+            message += f" {subject}"
+        message += f": {commit_message}"
+        if pr != -1:
+            message += f" (#{pr})"
+
+        messages.append((PRIORITY[commit_type], int(pr), message))
+
+    for message in sorted(messages, key=lambda x: (x[0], -x[1])):
+        print(message[2])
+
+
+if __name__ == "__main__":
+    main()