From d980808c9484ba4b996c3cf023d500c187bcca0a Mon Sep 17 00:00:00 2001 From: ludamad Date: Mon, 5 Feb 2024 00:08:39 +0000 Subject: [PATCH] refactor: simpler noir sync --- .github/workflows/mirror_noir_subrepo.yml | 84 ++++++++++++------- noir/.gitrepo | 4 +- scripts/git-subrepo/Intro.pod | 2 +- scripts/git-subrepo/doc/intro-to-subrepo.swim | 2 +- scripts/git-subrepo/ext/bashplus/bin/bash+ | 2 +- .../git-subrepo/ext/bashplus/lib/bash+.bash | 2 +- .../ext/test-more-bash/ext/bashplus/bin/bash+ | 2 +- .../ext/bashplus/lib/bash+.bash | 2 +- .../lib/git-subrepo.d/help-functions.bash | 2 +- scripts/git-subrepo/note/init-test | 2 +- .../note/recreate-rebase-conflict.sh | 4 +- .../subtree-rebase-fail-example/test.bash | 4 +- scripts/git-subrepo/note/test-subrepo-push.sh | 2 +- scripts/git-subrepo/note/test.sh | 2 +- .../pkg/bin/generate-help-functions.pl | 2 +- 15 files changed, 73 insertions(+), 45 deletions(-) diff --git a/.github/workflows/mirror_noir_subrepo.yml b/.github/workflows/mirror_noir_subrepo.yml index eaec104ed72d..c7b5e1938459 100644 --- a/.github/workflows/mirror_noir_subrepo.yml +++ b/.github/workflows/mirror_noir_subrepo.yml @@ -6,19 +6,32 @@ name: Mirror to noir repo # Don't allow multiple of these running at once: concurrency: group: ${{ github.workflow }} + cancel-in-progress: false on: push: branches: - - master - paths: - - 'noir/**' - - '!noir/.gitrepo' + - ad/chore/better-noir-sync + # paths: + # - 'noir/**' + # - '!noir/.gitrepo' jobs: mirror_repo: runs-on: ubuntu-latest steps: + - name: Get noir master's last sync commit + id: last_merge_hash + uses: actions/github-script@v7 + with: + script: | + // NOTE: more robust heuristic needed if aztecbot opens different kinds of PRs + const response = await github.rest.search.commits({ + q: 'author:AztecBot committer:web-flow repo:noir-lang/noir sort:committer-date' + }); + console.log(response.data.items); + return response.data.items[0].sha; + - name: Checkout uses: actions/checkout@v3 with: @@ -26,38 +39,53 @@ jobs: token: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }} # We push using git subrepo (https://github.com/ingydotnet/git-subrepo) - # with some logic to recover from squashed parent commits - # We push to subrepo, commit to master. The commit is needed - # to continue to replay. If we still hit issues such as this - # action failing due to upstream changes, a manual resolution - # PR with ./scripts/git_subrepo.sh pull will be needed. + # and push all Aztec commits as a single commit with metadata. - name: Push to branch run: | + set -xue # print commands SUBREPO_PATH=noir - BRANCH=aztec-packages + BRANCH=aztec2-packages # identify ourselves, needed to commit git config --global user.name AztecBot git config --global user.email tech@aztecprotocol.com - if ./scripts/git_subrepo.sh push $SUBREPO_PATH --branch=$BRANCH; then - git commit --amend -m "$(git log -1 --pretty=%B) [skip ci]" + BASE_NOIR_COMMIT=`git config --file=noir/.gitrepo subrepo.commit` + BASE_AZTEC_COMMIT=`git config --file=noir/.gitrepo subrepo.parent` + # Fix PR branch + git clone https://x-access-token:${{ secrets.AZTEC_BOT_GITHUB_TOKEN }}@github.com/noir-lang/noir.git noir-repo + cd noir-repo + git checkout $BRANCH || git checkout -b $BRANCH + git reset --hard "$BASE_NOIR_COMMIT" + # Reset our branch to our expected target + git push origin $BRANCH --force + cd .. + # Get the last merged PR + LAST_MERGED_PR_HASH=`gh pr list --repo=noir-lang/noir --state merged --head aztec-packages --json headRefOid --jq=.[0].headRefOid` + COMMIT_HEURISTIC=$(git log -p -S"$LAST_MERGED_PR_HASH" --reverse --source -- noir/.gitrepo | grep -m 1 '^commit' | awk '{print $2}' || true) + if [[ " $COMMIT_HEURISTIC" = "" ]] ; then + COMMIT_HEURISTIC=$BASE_AZTEC_COMMIT + fi + MESSAGE=$(git log --pretty=format:"%s" $COMMIT_HEURISTIC..HEAD -- noir/ ':!noir/.gitrepo' | grep -v 'git subrepo || true') + # Fix Aztec PR links + MESSAGE=$(echo "$MESSAGE" | sed -E 's/\(#([0-9]+)\)/(https:\/\/github.com\/AztecProtocol\/aztec-packages\/pull\/\1)/g') + git commit --allow-empty -m"chore: Sync to noir-lang/noir" -m"$MESSAGE" + COMMIT=$(git rev-parse HEAD) + # Now push to it with subrepo with computed commit messages + if ./scripts/git_subrepo.sh push $SUBREPO_PATH --squash --branch=$BRANCH; then + git reset $COMMIT + git commit --amend -am "$(git log -1 --pretty=%B) [skip ci]" git push else echo "Problems syncing noir. We may need to pull the subrepo." exit 1 fi - - - name: Checkout - uses: actions/checkout@v3 - with: - fetch-depth: 0 - repository: noir-lang/noir - token: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }} - - name: Create PR for Aztec Branch - continue-on-error: true - uses: repo-sync/pull-request@v2 - with: - github_token: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }} - pr_title: 'feat: Sync commits from `aztec-packages`' - pr_body: 'Development from [aztec-packages](https://github.com/AztecProtocol/aztec-packages).' - destination_branch: 'master' - source_branch: 'aztec-packages' + # Formatted for updating the PR, overrides for release-please commit message parsing + PR_BODY="""BEGIN_COMMIT_OVERRIDE + $MESSAGE + END_COMMIT_OVERRIDE""" + export GH_TOKEN="${{ secrets.AZTEC_BOT_GITHUB_TOKEN }}" + PR_URL=$(gh pr list --repo noir-lang/noir --head aztec2-packages --json url --jq ".[0].url") + if [[ "$PR_URL" == "" ]]; then + gh pr create --repo noir-lang/noir --title "feat: Sync from aztec-packages" --body "$PR_BODY" --base master --head aztec2-packages + else + gh pr edit "$PR_URL" --body "$PR_BODY" + fi \ No newline at end of file diff --git a/noir/.gitrepo b/noir/.gitrepo index aa2ef42f98fc..b2e388b2968e 100644 --- a/noir/.gitrepo +++ b/noir/.gitrepo @@ -6,7 +6,7 @@ [subrepo] remote = https://github.com/noir-lang/noir branch = aztec-packages - commit = 9a70040211d205f472aa31924649c2fef999eb49 - parent = 9c965a7c9e652dfeaba2f09152e5db287407473d + commit = 0205d3b4ad0cf5ffd775a43eb5af273a772cf138 + parent = 85324cc14b637f0c1347d6c21235b57b0628f6eb method = merge cmdver = 0.4.6 diff --git a/scripts/git-subrepo/Intro.pod b/scripts/git-subrepo/Intro.pod index a4d30716cc0f..656c5f32e0a1 100644 --- a/scripts/git-subrepo/Intro.pod +++ b/scripts/git-subrepo/Intro.pod @@ -78,7 +78,7 @@ Submodules tend to receive a lot of bad press. Here's some of it: =item * L -=item * L +=item * L =item * L diff --git a/scripts/git-subrepo/doc/intro-to-subrepo.swim b/scripts/git-subrepo/doc/intro-to-subrepo.swim index b4524ed21b0c..286663159ed0 100644 --- a/scripts/git-subrepo/doc/intro-to-subrepo.swim +++ b/scripts/git-subrepo/doc/intro-to-subrepo.swim @@ -61,7 +61,7 @@ article is about. Submodules tend to receive a lot of bad press. Here's some of it: * http://ayende.com/blog/4746/the-problem-with-git-submodules -* http://somethingsinistral.net/blog/git-submodules-are-probably-not-the-answer/ +* https://web.archive.org/web/20171101202911/http://somethingsinistral.net/blog/git-submodules-are-probably-not-the-answer/ * http://codingkilledthecat.wordpress.com/2012/04/28/why-your-company-shouldnt-use-git-submodules/ A quick recap of some of the good and bad things about submodules: diff --git a/scripts/git-subrepo/ext/bashplus/bin/bash+ b/scripts/git-subrepo/ext/bashplus/bin/bash+ index e0bc1f73913a..a691460242d9 100755 --- a/scripts/git-subrepo/ext/bashplus/bin/bash+ +++ b/scripts/git-subrepo/ext/bashplus/bin/bash+ @@ -5,7 +5,7 @@ # Copyright (c) 2013-2020 Ingy döt Net #------------------------------------------------------------------------------ -set -eu +set -e shopt -s compat31 &>/dev/null || true #------------------------------------------------------------------------------ diff --git a/scripts/git-subrepo/ext/bashplus/lib/bash+.bash b/scripts/git-subrepo/ext/bashplus/lib/bash+.bash index b52e17b55a62..dccdac08376f 100644 --- a/scripts/git-subrepo/ext/bashplus/lib/bash+.bash +++ b/scripts/git-subrepo/ext/bashplus/lib/bash+.bash @@ -2,7 +2,7 @@ # # Copyright (c) 2013-2020 Ingy döt Net -set -eu +set -e [[ ${BASHPLUS_VERSION-} ]] && return 0 diff --git a/scripts/git-subrepo/ext/test-more-bash/ext/bashplus/bin/bash+ b/scripts/git-subrepo/ext/test-more-bash/ext/bashplus/bin/bash+ index e0bc1f73913a..a691460242d9 100755 --- a/scripts/git-subrepo/ext/test-more-bash/ext/bashplus/bin/bash+ +++ b/scripts/git-subrepo/ext/test-more-bash/ext/bashplus/bin/bash+ @@ -5,7 +5,7 @@ # Copyright (c) 2013-2020 Ingy döt Net #------------------------------------------------------------------------------ -set -eu +set -e shopt -s compat31 &>/dev/null || true #------------------------------------------------------------------------------ diff --git a/scripts/git-subrepo/ext/test-more-bash/ext/bashplus/lib/bash+.bash b/scripts/git-subrepo/ext/test-more-bash/ext/bashplus/lib/bash+.bash index e0e15cfc49d7..2e1a70c20795 100644 --- a/scripts/git-subrepo/ext/test-more-bash/ext/bashplus/lib/bash+.bash +++ b/scripts/git-subrepo/ext/test-more-bash/ext/bashplus/lib/bash+.bash @@ -2,7 +2,7 @@ # # Copyright (c) 2013-2020 Ingy döt Net -set -eu +set -e [[ ${BASHPLUS_VERSION-} ]] && return 0 diff --git a/scripts/git-subrepo/lib/git-subrepo.d/help-functions.bash b/scripts/git-subrepo/lib/git-subrepo.d/help-functions.bash index 98e34dd272b0..123bb54cbb5c 100644 --- a/scripts/git-subrepo/lib/git-subrepo.d/help-functions.bash +++ b/scripts/git-subrepo/lib/git-subrepo.d/help-functions.bash @@ -2,7 +2,7 @@ # DO NOT EDIT. This file generated by pkg/bin/generate-help-functions.pl. -set -eu +set -e help:all() { cat <<'...' diff --git a/scripts/git-subrepo/note/init-test b/scripts/git-subrepo/note/init-test index 810dae14a4db..a20854de88ac 100755 --- a/scripts/git-subrepo/note/init-test +++ b/scripts/git-subrepo/note/init-test @@ -1,4 +1,4 @@ -#!/usr/bin/env bash +#!/bin/bash set -ex cat $0 # Show this script in the output diff --git a/scripts/git-subrepo/note/recreate-rebase-conflict.sh b/scripts/git-subrepo/note/recreate-rebase-conflict.sh index 6dc0a51e300d..21bf8365b75a 100644 --- a/scripts/git-subrepo/note/recreate-rebase-conflict.sh +++ b/scripts/git-subrepo/note/recreate-rebase-conflict.sh @@ -1,6 +1,6 @@ -#!/usr/bin/env bash +#!/bin/bash -set -eu +set -e set -x # Make a directory to work in: diff --git a/scripts/git-subrepo/note/subtree-rebase-fail-example/test.bash b/scripts/git-subrepo/note/subtree-rebase-fail-example/test.bash index 9118236848ad..df8b818cee61 100755 --- a/scripts/git-subrepo/note/subtree-rebase-fail-example/test.bash +++ b/scripts/git-subrepo/note/subtree-rebase-fail-example/test.bash @@ -1,4 +1,4 @@ -#!/usr/bin/env bash +#!/bin/bash set -ex @@ -24,6 +24,6 @@ git clone repo1 repo3 git subrepo clone ../repo2 subrepo bash git rebase -i HEAD^ - git log -p + git log -p ls ) diff --git a/scripts/git-subrepo/note/test-subrepo-push.sh b/scripts/git-subrepo/note/test-subrepo-push.sh index 13b76e47fc82..afceb5efa92f 100644 --- a/scripts/git-subrepo/note/test-subrepo-push.sh +++ b/scripts/git-subrepo/note/test-subrepo-push.sh @@ -1,4 +1,4 @@ -#!/usr/bin/env bash +#!/bin/bash set -ex diff --git a/scripts/git-subrepo/note/test.sh b/scripts/git-subrepo/note/test.sh index d27548f0937f..fae278795d1c 100755 --- a/scripts/git-subrepo/note/test.sh +++ b/scripts/git-subrepo/note/test.sh @@ -1,4 +1,4 @@ -#!/usr/bin/env bash +#!/bin/bash set -x diff --git a/scripts/git-subrepo/pkg/bin/generate-help-functions.pl b/scripts/git-subrepo/pkg/bin/generate-help-functions.pl index 5dd949e6ed90..40b6af8f2b38 100644 --- a/scripts/git-subrepo/pkg/bin/generate-help-functions.pl +++ b/scripts/git-subrepo/pkg/bin/generate-help-functions.pl @@ -33,7 +33,7 @@ sub write_start { # DO NOT EDIT. This file generated by pkg/bin/generate-help-functions.pl. -set -eu +set -e ... }