From 169d3562158687010fe1423ff897402dc10e4951 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Wed, 27 Aug 2025 15:38:21 +0200 Subject: [PATCH 1/6] sync-git-gui: rename it, to form the basis of a generic workflow In the upcoming work to support a project other than Git, we will need to synchronize a different upstream repository to a different "PR repository". To that end, let's consolidate `sync-git-gui` and `sync-gitster-git` into a single, matrix workflow that will continue to synchronize the Git GUI and the fine-grained branches from `gitster/git` to `gitgitgadget/git`, but can be overridden in another fork of `gitgitgadget-workflows` via a repository variable that contains the project config to perform the equivalent job of synchronizing upstream branches into the repository where GitGitGadget handles the Pull Requests for said project. This is step 1 to support that: renaming the `sync-git-gui` workflow to `sync-upstream-branches`. Subsequent commits will transmogrify that workflow into the desired shape. Signed-off-by: Johannes Schindelin --- .../workflows/{sync-git-gui.yml => sync-upstream-branches.yml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{sync-git-gui.yml => sync-upstream-branches.yml} (100%) diff --git a/.github/workflows/sync-git-gui.yml b/.github/workflows/sync-upstream-branches.yml similarity index 100% rename from .github/workflows/sync-git-gui.yml rename to .github/workflows/sync-upstream-branches.yml From 7a1ca712fd85f737d56fad59d810e178a4bec725 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 29 Aug 2025 15:37:38 +0200 Subject: [PATCH 2/6] sync-upstream-branches: turn the workflow into a matrix The workflow still only handles the `git-gui` branches, but is now almost generic enough to become a matrix workflow that also handles the `gitster/git` synchronization. Signed-off-by: Johannes Schindelin --- .github/workflows/sync-upstream-branches.yml | 32 ++++++++++++-------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/.github/workflows/sync-upstream-branches.yml b/.github/workflows/sync-upstream-branches.yml index 5f49ac5..d9aa636 100644 --- a/.github/workflows/sync-upstream-branches.yml +++ b/.github/workflows/sync-upstream-branches.yml @@ -1,15 +1,10 @@ -name: sync-git-gui-branches +name: sync-upstream-branches on: schedule: - cron: '31 22 * * *' workflow_dispatch: -env: - SOURCE_REPOSITORY: j6t/git-gui - TARGET_REPOSITORY: gitgitgadget/git - TARGET_REF_NAMESPACE: git-gui/ - # We want to limit queuing to a single workflow run i.e. if there is already # an active workflow run and a queued one, queue another one canceling the # already queued one. @@ -17,15 +12,26 @@ concurrency: group: ${{ github.workflow }} jobs: - sync-git-gui-branches: + sync-upstream-branches: runs-on: ubuntu-latest + strategy: + matrix: + spec: + - sourceRepo: j6t/git-gui + targetRepo: gitgitgadget/git + targetRefNamespace: git-gui/ + steps: - name: check which refs need to be synchronized uses: actions/github-script@v7 id: check with: script: | - const [targetRepoOwner, targetRepoName] = process.env.TARGET_REPOSITORY.split('/') + const sourceRepo = ${{ toJSON(matrix.spec.sourceRepo) }} + const targetRepo = ${{ toJSON(matrix.spec.targetRepo) }} + const targetRefNamespace = ${{ toJSON(matrix.spec.targetRefNamespace) }} || '' + + const [targetRepoOwner, targetRepoName] = targetRepo.split('/') core.setOutput('target-repo-owner', targetRepoOwner) core.setOutput('target-repo-name', targetRepoName) @@ -72,10 +78,10 @@ jobs: } } - const sourceRefs = await getRefs(process.env.SOURCE_REPOSITORY) - const targetRefs = await getRefs(process.env.TARGET_REPOSITORY, process.env.TARGET_REF_NAMESPACE) + const sourceRefs = await getRefs(sourceRepo) + const targetRefs = await getRefs(targetRepo, targetRefNamespace) - const targetPrefix = `refs/heads/${process.env.TARGET_REF_NAMESPACE}` + const targetPrefix = `refs/heads/${targetRefNamespace}` const refspecs = [] const toFetch = new Set() @@ -138,7 +144,7 @@ jobs: set -ex git init --bare - git remote add source "${{ github.server_url }}/$SOURCE_REPOSITORY" + git remote add source '${{ github.server_url }}/${{ matrix.spec.sourceRepo }}' # pretend to be a partial clone git config remote.source.promisor true git config remote.source.partialCloneFilter blob:none @@ -151,4 +157,4 @@ jobs: # push the commits printf '%s' '${{ steps.check.outputs.refspec }}' | xargs -d ' ' -r git -c http.extraHeader='${{ steps.auth.outputs.header }}' \ - push "${{ github.server_url }}/$TARGET_REPOSITORY" + push '${{ github.server_url }}/${{ matrix.spec.targetRepo }}' From 6cc9af24c03a18888b674dfd2c76939c8abb8efc Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 29 Aug 2025 15:49:33 +0200 Subject: [PATCH 3/6] sync-upstream-branches: adopt a more functional style This still does the exact same thing, but avoids storing intermediate states of the `data` array's transformation. Signed-off-by: Johannes Schindelin --- .github/workflows/sync-upstream-branches.yml | 33 +++++++++----------- 1 file changed, 14 insertions(+), 19 deletions(-) diff --git a/.github/workflows/sync-upstream-branches.yml b/.github/workflows/sync-upstream-branches.yml index d9aa636..fa0bd38 100644 --- a/.github/workflows/sync-upstream-branches.yml +++ b/.github/workflows/sync-upstream-branches.yml @@ -44,27 +44,22 @@ jobs: for (;;) { try { const [owner, repo] = repository.split('/') - let { data } = await github.rest.git.listMatchingRefs({ - owner, - repo, - ref: 'heads/' - }) - - data = data.filter(e => { - if (!e.ref.startsWith('refs/heads/')) return false - e.name = e.ref.slice(11) - return true - }) - - if (stripRefsPrefix) { - data = data.filter(e => { - if (!e.name.startsWith(stripRefsPrefix)) return false - e.name = e.name.slice(stripRefsPrefix.length) + return ( + await github.rest.git.listMatchingRefs({ + owner, + repo, + ref: 'heads/' + }) + ).data + .filter((e) => { + if (!e.ref.startsWith('refs/heads/')) return false + e.name = e.ref.slice(11) + if (stripRefsPrefix) { + if (!e.name.startsWith(stripRefsPrefix)) return false + e.name = e.name.slice(stripRefsPrefix.length) + } return true }) - } - - return data .sort((a, b) => a.ref.localeCompare(b.ref)) } catch (e) { if (e?.status !== 502) throw e From 265711cf69890fa21079daab4849f42dbc56d87b Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 29 Aug 2025 15:40:31 +0200 Subject: [PATCH 4/6] sync-upstream-branches: take over the job of the `sync-gitster-git` workflow With a few touch-ups, the workflow can become generic enough to handle also the synchronization of `gitster/git`'s branches to `gitgitgadget/git`. The overall goal, of course, is to prepare for supporting projects other than Git, via configurations stored in repository variables. That's the task of the next commit(s). Signed-off-by: Johannes Schindelin --- .github/workflows/sync-gitster-git.yml | 140 ------------------- .github/workflows/sync-upstream-branches.yml | 8 ++ 2 files changed, 8 insertions(+), 140 deletions(-) delete mode 100644 .github/workflows/sync-gitster-git.yml diff --git a/.github/workflows/sync-gitster-git.yml b/.github/workflows/sync-gitster-git.yml deleted file mode 100644 index 6a3af09..0000000 --- a/.github/workflows/sync-gitster-git.yml +++ /dev/null @@ -1,140 +0,0 @@ -name: sync-gitster-git-branches - -on: - schedule: - - cron: '17 6 * * *' - workflow_dispatch: - -env: - SOURCE_REPOSITORY: gitster/git - TARGET_REPOSITORY: gitgitgadget/git - -# We want to limit queuing to a single workflow run i.e. if there is already -# an active workflow run and a queued one, queue another one canceling the -# already queued one. -concurrency: - group: ${{ github.workflow }} - -jobs: - sync-gitster-git-branches: - runs-on: ubuntu-latest - steps: - - name: check which refs need to be synchronized - uses: actions/github-script@v7 - id: check - with: - script: | - const [targetRepoOwner, targetRepoName] = process.env.TARGET_REPOSITORY.split('/') - core.setOutput('target-repo-owner', targetRepoOwner) - core.setOutput('target-repo-name', targetRepoName) - - const sleep = async (milliseconds) => { - return new Promise(resolve => setTimeout(resolve, milliseconds)) - } - - const getRefs = async (repository) => { - let attemptCounter = 1 - for (;;) { - try { - const [owner, repo] = repository.split('/') - const { data } = await github.rest.git.listMatchingRefs({ - owner, - repo, - // We want to match `maint-*` as well as `[a-z][a-z]/*` - // sadly, this is not possible via GitHub's REST API, - // hence we do it below via the `filter()` call. - ref: 'heads/' - }) - return data - .filter(e => e.ref.match(/^refs\/heads\/(maint-\d|[a-z][a-z]\/)/)) - .sort((a, b) => a.ref.localeCompare(b.ref)) - } catch (e) { - if (e?.status !== 502) throw e - } - - if (++attemptCounter > 10) throw new Error('Giving up listing refs after 10 attempts') - - const seconds = attemptCounter * attemptCounter + 15 * Math.random() - core.info(`Encountered a Server Error; retrying in ${seconds} seconds`) - await sleep(1000 * seconds) - } - } - - const sourceRefs = await getRefs(process.env.SOURCE_REPOSITORY) - const targetRefs = await getRefs(process.env.TARGET_REPOSITORY) - - const refspecs = [] - const toFetch = new Set() - for (let i = 0, j = 0; i < sourceRefs.length || j < targetRefs.length; ) { - const compare = i >= sourceRefs.length - ? +1 - : j >= targetRefs.length - ? -1 - : sourceRefs[i].ref.localeCompare(targetRefs[j].ref) - if (compare > 0) { - // no source ref => delete target ref - refspecs.push(`:${targetRefs[j].ref}`) - j++ - } else if (compare < 0) { - // no corresponding target ref yet => push source ref (new) - const sha = sourceRefs[i].object.sha - toFetch.add(sha) - refspecs.push(`${sha}:${sourceRefs[i].ref}`) - i++ - } else { - // the sourceRef's name matches the targetRef's - if (sourceRefs[i].object.sha !== targetRefs[j].object.sha) { - // target ref needs updating - const sha = sourceRefs[i].object.sha - toFetch.add(sha) - refspecs.push(`+${sha}:${sourceRefs[i].ref}`) - } - i++ - j++ - } - } - - core.setOutput('refspec', refspecs.join(' ')) - targetRefs.forEach((e) => toFetch.delete(e.object.sha)) - core.setOutput('to-fetch', [...toFetch].join(' ')) - - name: obtain installation token - if: steps.check.outputs.refspec != '' - uses: actions/create-github-app-token@v2 - id: token - with: - app-id: ${{ secrets.GITGITGADGET_GITHUB_APP_ID }} - private-key: ${{ secrets.GITGITGADGET_GITHUB_APP_PRIVATE_KEY }} - owner: ${{ steps.check.outputs.target-repo-owner }} - repositories: ${{ steps.check.outputs.target-repo-name }} - - name: set authorization header - if: steps.check.outputs.refspec != '' - uses: actions/github-script@v7 - id: auth - with: - script: | - // Sadly, `git push` does not work with 'Authorization: Bearer ', therefore - // we have to use the `Basic` variant - const auth = Buffer.from('PAT:${{ steps.token.outputs.token }}').toString('base64') - core.setSecret(auth) - core.setOutput('header', `Authorization: Basic ${auth}`) - - name: sync - if: steps.check.outputs.refspec != '' - shell: bash - run: | - set -ex - git init --bare - - git remote add source "${{ github.server_url }}/$SOURCE_REPOSITORY" - # pretend to be a partial clone - git config remote.source.promisor true - git config remote.source.partialCloneFilter blob:none - - # fetch some commits - printf '%s' '${{ steps.check.outputs.to-fetch }}' | - xargs -d ' ' -r git fetch --depth 10000 source - rm -f .git/shallow - - # push the commits - printf '%s' '${{ steps.check.outputs.refspec }}' | - xargs -d ' ' -r git -c http.extraHeader='${{ steps.auth.outputs.header }}' \ - push "${{ github.server_url }}/$TARGET_REPOSITORY" diff --git a/.github/workflows/sync-upstream-branches.yml b/.github/workflows/sync-upstream-branches.yml index fa0bd38..f1c39de 100644 --- a/.github/workflows/sync-upstream-branches.yml +++ b/.github/workflows/sync-upstream-branches.yml @@ -20,6 +20,9 @@ jobs: - sourceRepo: j6t/git-gui targetRepo: gitgitgadget/git targetRefNamespace: git-gui/ + - sourceRepo: gitster/git + targetRepo: gitgitgadget/git + sourceRefRegex: "^refs/heads/(maint-\\d|[a-z][a-z]/)" steps: - name: check which refs need to be synchronized @@ -28,6 +31,7 @@ jobs: with: script: | const sourceRepo = ${{ toJSON(matrix.spec.sourceRepo) }} + const sourceRefRegexp = ((p) => p ? new RegExp(p) : null)(${{ toJSON(matrix.spec.sourceRefRegex) }}) const targetRepo = ${{ toJSON(matrix.spec.targetRepo) }} const targetRefNamespace = ${{ toJSON(matrix.spec.targetRefNamespace) }} || '' @@ -48,10 +52,14 @@ jobs: await github.rest.git.listMatchingRefs({ owner, repo, + // We cannot match `source-ref-regex` as freely as we + // want with GitHub's REST API, hence we do it below via + // the `filter()` call. ref: 'heads/' }) ).data .filter((e) => { + if (sourceRefRegexp && !sourceRefRegexp.test(e.ref)) return false if (!e.ref.startsWith('refs/heads/')) return false e.name = e.ref.slice(11) if (stripRefsPrefix) { From ca0a54dc3098c025753d8a74df244d2662b66bcf Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Sat, 30 Aug 2025 13:18:43 +0200 Subject: [PATCH 5/6] sync-git-mailing-list-mirror: start making the workflow more generic I want to use the same workflow in a fork to synchronize _Cygwin's_ mailing list mirror; Let's rename the workflow and its job so that it stops claiming to synchronize _Git's_ mailing list mirror. Signed-off-by: Johannes Schindelin --- ...mailing-list-mirror.yml => sync-mailing-list-mirror.yml} | 6 +++--- README.md | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) rename .github/workflows/{sync-git-mailing-list-mirror.yml => sync-mailing-list-mirror.yml} (94%) diff --git a/.github/workflows/sync-git-mailing-list-mirror.yml b/.github/workflows/sync-mailing-list-mirror.yml similarity index 94% rename from .github/workflows/sync-git-mailing-list-mirror.yml rename to .github/workflows/sync-mailing-list-mirror.yml index 89d4bd6..3447beb 100644 --- a/.github/workflows/sync-git-mailing-list-mirror.yml +++ b/.github/workflows/sync-mailing-list-mirror.yml @@ -1,4 +1,4 @@ -name: sync-git-mailing-list-mirror +name: sync-mailing-list-mirror on: workflow_dispatch: @@ -11,11 +11,11 @@ env: TARGET_GITHUB_REPOSITORY: gitgitgadget/git-mailing-list-mirror concurrency: - group: sync-git-mailing-list-mirror + group: sync-mailing-list-mirror cancel-in-progress: true jobs: - sync-git-mailing-list-mirror: + sync-mailing-list-mirror: runs-on: ubuntu-latest permissions: contents: write diff --git a/README.md b/README.md index ac8a925..28d4ef6 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,6 @@ This repository contains GitHub workflows, i.e. automated tasks, that keep [GitG ## Keeping the mirror of the Git mailing list up to date -The `sync-git-mailing-list.yml` workflow keeps the mirror at https://github.com/gitgitgadget/git-mailing-list of the Git mailing list mirror at https://lore.kernel.org/git up to date. Since that mirror chunks the archive by epochs, this mirror fetches each epoch into its own branch: the oldest epoch into `lore-0`, the next one into `lore-1`, etc. +The `sync-mailing-list-mirror.yml` workflow keeps the mirror at https://github.com/gitgitgadget/git-mailing-list of the Git mailing list mirror at https://lore.kernel.org/git up to date. Since that mirror chunks the archive by epochs, this mirror fetches each epoch into its own branch: the oldest epoch into `lore-0`, the next one into `lore-1`, etc. Previously, this workflow lived in the `git-mailing-list` repository in the `sync` branch, which was the default branch because scheduled workflows _must_ live in the default branch. From 3c327fc5eeefb34f2b26ee5424541e1ebb177724 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Sat, 30 Aug 2025 13:19:55 +0200 Subject: [PATCH 6/6] sync-mailing-list-mirror: use more elegant cron definition Rather than hard-coding the exact minutes at which the workflow should run, just specify that it should run every five minutes, and leave it to GitHub to figure out the specifics. Signed-off-by: Johannes Schindelin --- .github/workflows/sync-mailing-list-mirror.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sync-mailing-list-mirror.yml b/.github/workflows/sync-mailing-list-mirror.yml index 3447beb..b08b856 100644 --- a/.github/workflows/sync-mailing-list-mirror.yml +++ b/.github/workflows/sync-mailing-list-mirror.yml @@ -3,7 +3,7 @@ name: sync-mailing-list-mirror on: workflow_dispatch: schedule: - - cron: "2,7,12,17,22,27,32,37,42,47,52,57 * * * *" + - cron: "*/5 * * * *" env: LORE_EPOCH: 1 # also adjust SOURCE_REPOSITORY