From 3d176e3d5dfe70cb2f95afa2e590119d85e62603 Mon Sep 17 00:00:00 2001
From: Alex Crichton <alex@alexcrichton.com>
Date: Thu, 21 Nov 2024 14:57:32 -0700
Subject: [PATCH] Rewrite the update script in Python (#112)

* Rewrite the update script in Python

With various updates to this script such as having a different upstream
for `memory64` and otherwise including the `wasm-3.0` test suite this
script was getting relatively complicated which is typically not a great
situation for any bash script to be in. Instead rewrite this script in
Python to avoid many of bash's pitfalls and be a bit more
readable/reviewable/modifiable.

* Review comments
---
 .github/workflows/autoupdate.yml |   2 +-
 README.md                        |   2 +-
 repos/README.md                  |   3 -
 update-testsuite.py              | 229 +++++++++++++++++++++++++++++++
 update-testsuite.sh              | 209 ----------------------------
 5 files changed, 231 insertions(+), 214 deletions(-)
 delete mode 100644 repos/README.md
 create mode 100755 update-testsuite.py
 delete mode 100755 update-testsuite.sh

diff --git a/.github/workflows/autoupdate.yml b/.github/workflows/autoupdate.yml
index 3fbc4bb..28afd99 100644
--- a/.github/workflows/autoupdate.yml
+++ b/.github/workflows/autoupdate.yml
@@ -29,7 +29,7 @@ jobs:
       - run: |
           git config --global user.name 'WebAssembly/testsuite auto-update'
           git config --global user.email 'github-actions@users.noreply.github.com'
-          ./update-testsuite.sh
+          ./update-testsuite.py
 
       # If the current HEAD is different then a commit was made, so make a PR.
       - run: |
diff --git a/README.md b/README.md
index 90c453f..b96fdbe 100644
--- a/README.md
+++ b/README.md
@@ -20,5 +20,5 @@ This repository is updated weekly on Wednesday via automated pull requests.
 Maintainers can also
 [manually trigger an update](https://github.com/WebAssembly/testsuite/actions/workflows/autoupdate.yml).
 
-Contributors can update tests by running the `./update-testsuite.sh` script and
+Contributors can update tests by running the `./update-testsuite.py` script and
 making a pull request.
diff --git a/repos/README.md b/repos/README.md
deleted file mode 100644
index 0fd8bc3..0000000
--- a/repos/README.md
+++ /dev/null
@@ -1,3 +0,0 @@
-The spec and proposal repositories will be cloned in this directory by the
-`update-testsuite.sh` script. Don't apply local changes to these repositories,
-as the script may destroy them.
diff --git a/update-testsuite.py b/update-testsuite.py
new file mode 100755
index 0000000..9a8a5ac
--- /dev/null
+++ b/update-testsuite.py
@@ -0,0 +1,229 @@
+#!/usr/bin/env python3
+
+"""
+Script to automatically update all tests in this repository based on the
+status of the upstream repositories themselves. This will prepare a git
+submodule-of-sorts (not literally) in `repos` with all the upstream
+repos fetched into that one location. Tests will be diff'd against the merged
+version of the upstream spec repo and the proposal repo, and if there's a
+difference then the test is included.
+"""
+
+import os
+import shutil
+import subprocess
+import sys
+
+
+class GitError(RuntimeError):
+    def __init__(self, output, args):
+        self.output = output
+        self.args = args
+
+    def __str__(self):
+        cmd = ' '.join(self.args)
+        desc = f'failed to run: git {cmd}'
+        desc += f'\n\treturncode: {self.output.returncode}'
+        if len(self.output.stdout) > 0:
+            stdout = self.output.stdout.replace('\n', '\n\t\t')
+            desc += f'\n\tstdout:\n\t\t{stdout}'
+        return desc
+
+
+def git(*args, quiet=False):
+    """
+    Helper to run a `git` command and handle the output/exit code in an
+    ergonomic fashion.
+    """
+    if not quiet:
+        print('running: git', *args)
+    ret = subprocess.run(['git', *args], stdout=subprocess.PIPE, text=True)
+    if ret.returncode != 0:
+        raise GitError(ret, args)
+    return ret.stdout.strip()
+
+
+class Repo:
+    def __init__(self, repo, branch = 'main', upstream = None):
+        self.repo = repo
+        if branch == 'main':
+            self.dir = repo
+        else:
+            self.dir = branch
+        self.branch = branch
+        self.upstream = upstream
+
+    def __str__(self):
+        repo = f'WebAssembly/{self.repo}'
+        if self.branch != 'main':
+            repo += f'@{self.branch}'
+        return repo
+
+    def update(self):
+        """
+        Fetch the latest revision from this repository and store the revision
+        within `self` of what was found.
+        """
+        Repo.git('fetch', self.url(), self.branch)
+        self.rev = Repo.git('rev-parse', 'FETCH_HEAD', quiet=True)
+
+    def url(self):
+        return f'https://github.com/WebAssembly/{self.repo}'
+
+    @staticmethod
+    def git(*args, **kwargs):
+        """
+        Helper to run a `git` command within the `repos` repository.
+        """
+        return git('-C', 'repos', *args, **kwargs)
+
+    def checkout_merge(self, spec=None):
+        """
+        Check out this repository to `repos` with a merge against `spec`
+        if specified.
+
+        Returns `False` if the merge fails.
+        """
+        Repo.git('checkout', '-B', 'try-merge')
+        Repo.git('reset', '--hard', self.rev)
+
+        if spec is None:
+            return True
+
+        try:
+            # Attempt to merge with the `spec` repository
+            Repo.git('merge', '-q', spec.rev, '-m', 'merge')
+        except GitError:
+            # If the merge failed try to ignore merge conflicts in non-test
+            # directories as we don't care about those changes
+            non_tests = ':(exclude)test/'
+            Repo.git('checkout', '--ours', non_tests)
+            Repo.git('add', non_tests)
+            try:
+                Repo.git('-c', 'core.editor=true', 'merge', '--continue')
+            except GitError:
+                # If all that failed then the merge couldn't be done.
+                Repo.git('merge', '--abort')
+                return False
+        self.merged_rev = Repo.git('rev-parse', 'HEAD', quiet=True)
+        return True
+
+    def list_tests(self):
+        """
+        Return a list-of-triples where each triple is
+
+          (path_to_test, git_path_of_test, destination_path_of_test)
+
+        This is used to run diffs and copy the test to its final location.
+        """
+        tests = []
+        for subdir in ['core', 'legacy']:
+            for root, dirs, files in os.walk(f'repos/test/{subdir}'):
+                for file in files:
+                    path = os.path.join(root, file)
+                    repo_path = os.path.relpath(path, 'repos')
+                    ext = os.path.splitext(path)[1]
+                    if ext != '.wast':
+                        continue
+                    dst = os.path.basename(path)
+                    if subdir == 'legacy':
+                        dst = 'legacy/' + dst
+                    tests.append((path, repo_path, dst))
+        return tests
+
+
+def main():
+    spec = Repo('spec')
+    spec3 = Repo('spec', branch='wasm-3.0')
+
+    repos = [
+        spec3,
+        Repo('threads'),
+        Repo('exception-handling'),
+        Repo('gc'),
+        Repo('tail-call'),
+        Repo('annotations'),
+        Repo('function-references'),
+        Repo('memory64', upstream=spec3),
+        Repo('extended-const'),
+        Repo('multi-memory'),
+        Repo('relaxed-simd'),
+        Repo('custom-page-sizes'),
+        Repo('wide-arithmetic'),
+    ]
+
+    # Make sure that `repos` is a git repository
+    if not os.path.isdir('repos'):
+        git('init', 'repos')
+
+    failed_merges = []
+    updated = []
+
+    # Update the spec itself, reset to the latest version of the spec, and then
+    # copy all files from the upstream spec tests into this repository's own
+    # suite of tests to run.
+    spec.update()
+    spec.checkout_merge()
+    tests = []
+    for path, _repo_path, dst in spec.list_tests():
+        shutil.copyfile(path, dst)
+        tests.append(dst)
+    git('add', *tests, quiet=True)
+    status = git('status', '-s', *tests, quiet=True)
+    if len(status) > 0:
+        updated.append(spec)
+
+    # Process all upstream repositories and proposals.
+    for repo in repos:
+        # Repositories may not be mergable with the upstream spec repository in
+        # which case we skip them and print an informational message at the end.
+        repo.update()
+        if not repo.checkout_merge(spec):
+            failed_merges.append(repo)
+            continue
+
+        # Blow away this proposal's list of tests if it exists.
+        dstdir = f'proposals/{repo.dir}'
+        if os.path.isdir(dstdir):
+            shutil.rmtree(dstdir)
+
+        # For all tests in this proposal run a diff against the upstream
+        # revision. If the diff is non-empty then include this test by copying
+        # it to its destination.
+        for path, repo_path, dst in spec.list_tests():
+            upstream = repo.upstream or spec
+
+            diff = repo.git('diff', upstream.rev, repo.merged_rev, '--', repo_path, quiet=True)
+            if len(diff) == 0:
+                continue
+
+            dst = os.path.join(dstdir, dst)
+            os.makedirs(os.path.dirname(dst), exist_ok=True)
+            shutil.copyfile(path, dst)
+
+        # If anything changed, then this is an updated proposal, and take note
+        # of that for later.
+        git('add', dstdir)
+        status = git('status', '-s', dstdir)
+        if len(status) > 0:
+            updated.append(repo)
+
+    for repo in failed_merges:
+        print('!! failed to update:', repo.url())
+
+    # If anything was updated, make a commit message indicating as such.
+    if len(updated) == 0:
+        print('No spec changes found, not creating a new commit')
+        return 0
+    message = 'Update repos:\n\n'
+    for repo in updated:
+        message += f'  {repo.dir}:\n'
+        message += f'    {repo.url()}/commit/{repo.rev}\n'
+    message += '\n'
+    message += 'This change was automatically generated by `update-testsuite.py`'
+    git('commit', '-a', '-m', message)
+    return 0
+
+
+if __name__ == '__main__':
+    sys.exit(main())
diff --git a/update-testsuite.sh b/update-testsuite.sh
deleted file mode 100755
index 1574535..0000000
--- a/update-testsuite.sh
+++ /dev/null
@@ -1,209 +0,0 @@
-#!/bin/bash
-# Update tests based on upstream repositories.
-set -e
-set -u
-set -o pipefail
-
-non_tests=":(exclude)test/"
-
-repos='
-  spec
-  threads
-  exception-handling
-  gc
-  tail-call
-  annotations
-  function-references
-  memory64
-  extended-const
-  multi-memory
-  relaxed-simd
-  custom-page-sizes
-  wide-arithmetic
-  wasm-3.0
-'
-
-log_and_run() {
-    echo ">>" $*
-    if ! $*; then
-        echo "sub-command failed: $*"
-        exit
-    fi
-}
-
-try_log_and_run() {
-    echo ">>" $*
-    $*
-}
-
-pushdir() {
-    pushd $1 >/dev/null || exit
-}
-
-popdir() {
-    popd >/dev/null || exit
-}
-
-update_repo() {
-    local dir=$1
-    local branch=main
-    local repo=$dir
-    if [ "${repo}" == "wasm-3.0" ]; then
-      branch="wasm-3.0"
-      repo="spec"
-    fi
-    pushdir repos
-        if [ -d ${dir} ]; then
-            log_and_run git -C ${dir} fetch origin
-            log_and_run git -C ${dir} reset origin/${branch} --hard
-        else
-            log_and_run git clone https://github.com/WebAssembly/${repo} ${dir} --branch ${branch}
-        fi
-
-        # Add upstream spec as "spec" remote.
-        if [ "${dir}" != "spec" ]; then
-            pushdir ${dir}
-                if ! git remote | grep spec >/dev/null; then
-                    log_and_run git remote add spec https://github.com/WebAssembly/spec
-                fi
-
-                log_and_run git fetch spec
-            popdir
-        fi
-    popdir
-}
-
-set_upstream() {
-    local repo=$1
-    upstream="main"
-    [ "${repo}" == "memory64" ] && upstream="wasm-3.0"
-    echo "set_upstream $upstream"
-}
-
-merge_with_spec() {
-    local repo=$1
-
-    [ "${repo}" == "spec" ] && return
-
-    set_upstream ${repo}
-
-    local head_ref=origin/HEAD
-    [ "${repo}" == "wasm-3.0" ] && head_ref=origin/wasm-3.0
-
-    pushdir repos/${repo}
-        # Create and checkout "try-merge" branch.
-        if ! git branch | grep try-merge >/dev/null; then
-            log_and_run git branch try-merge $head_ref
-        fi
-        log_and_run git checkout try-merge
-
-        # Attempt to merge with upstream branch in spec repo.
-        log_and_run git reset $head_ref --hard
-        try_log_and_run git merge -q spec/$upstream -m "merged"
-        if [ $? -ne 0 ]; then
-            # Ignore merge conflicts in non-test directories.
-            # We don't care about those changes.
-            try_log_and_run git checkout --ours ${non_tests}
-            try_log_and_run git add ${non_tests}
-            try_log_and_run git -c core.editor=true merge --continue
-            if [ $? -ne 0 ]; then
-                git merge --abort
-                popdir
-                return 1
-            fi
-        fi
-    popdir
-    return 0
-}
-
-
-echo -e "Update repos\n" > commit_message
-
-failed_repos=
-any_updated=0
-
-for repo in ${repos}; do
-    echo "++ updating ${repo}"
-    update_repo ${repo}
-
-    if ! merge_with_spec ${repo}; then
-        echo -e "!! error merging ${repo}, skipping\n"
-        failed_repos="${failed_repos} ${repo}"
-        continue
-    fi
-
-    if [ "${repo}" = "spec" ]; then
-        wast_dir=.
-        log_and_run cp $(find repos/${repo}/test/core -name \*.wast) ${wast_dir}
-    else
-        wast_dir=proposals/${repo}
-        # Start by removing any existing test files for this proposal.
-        log_and_run rm -rf ${wast_dir}/
-        mkdir -p ${wast_dir}
-
-        # Checkout the corresponding upstream branch in the spec repo
-        set_upstream ${repo}
-        pushdir repos/spec
-          try_log_and_run git checkout origin/${upstream}
-        popdir
-
-        # Don't add tests from proposal that are the same as spec.
-        pushdir repos/${repo}
-            for new in $(find test/core -name \*.wast); do
-                old=../../repos/spec/${new}
-                if [[ ! -f ${old} ]] || ! diff ${old} ${new} >/dev/null; then
-                    log_and_run cp ${new} ../../${wast_dir}/
-                fi
-            done
-            for new in $(find test/legacy -name \*.wast); do
-                old=../../repos/spec/${new}
-                if [[ ! -f ${old} ]] || ! diff ${old} ${new} >/dev/null; then
-                    mkdir -p ../../${wast_dir}/legacy/
-                    log_and_run cp ${new} ../../${wast_dir}/legacy/
-                fi
-            done
-        popdir
-    fi
-
-    # Check whether any files were removed.
-    for old in $(find ${wast_dir} -maxdepth 1 -name \*.wast); do
-      new=$(find repos/${repo}/test/core -name ${old##*/})
-      if [[ ! -f ${new} ]]; then
-          log_and_run git rm ${old}
-      fi
-    done
-
-    # Check whether any files were updated.
-    if [ $(git status -s ${wast_dir} | wc -l) -ne 0 ]; then
-        log_and_run git add ${wast_dir}
-
-        branch=main
-        dir=${repo}
-        if [ "${repo}" == "wasm-3.0" ]; then
-          branch="wasm-3.0"
-          repo="spec"
-        fi
-        repo_sha=$(git -C repos/${dir} log --max-count=1 --oneline origin/$branch | sed -e 's/ .*//')
-        echo "  ${dir}:" >> commit_message
-        echo "    https://github.com/WebAssembly/${repo}/commit/${repo_sha}" >> commit_message
-        any_updated=1
-    fi
-
-    echo -e "-- ${repo}\n"
-done
-
-if [ -n "${failed_repos}" ]; then
-  echo "!! failed to update repos: ${failed_repos}"
-fi
-
-if [ "$any_updated" = "0" ]; then
-  echo "no tests were updated from upstream repositories"
-else
-  echo "" >> commit_message
-  echo "This change was automatically generated by \`update-testsuite.sh\`" >> commit_message
-  git commit -a -F commit_message
-  # git push
-fi
-
-
-echo "done"