diff --git a/.github/workflows/sync-gitster-git.yml b/.github/workflows/sync-gitster-git.yml deleted file mode 100644 index 6a3af09..0000000 --- a/.github/workflows/sync-gitster-git.yml +++ /dev/null @@ -1,140 +0,0 @@ -name: sync-gitster-git-branches - -on: - schedule: - - cron: '17 6 * * *' - workflow_dispatch: - -env: - SOURCE_REPOSITORY: gitster/git - TARGET_REPOSITORY: gitgitgadget/git - -# We want to limit queuing to a single workflow run i.e. if there is already -# an active workflow run and a queued one, queue another one canceling the -# already queued one. -concurrency: - group: ${{ github.workflow }} - -jobs: - sync-gitster-git-branches: - runs-on: ubuntu-latest - steps: - - name: check which refs need to be synchronized - uses: actions/github-script@v7 - id: check - with: - script: | - const [targetRepoOwner, targetRepoName] = process.env.TARGET_REPOSITORY.split('/') - core.setOutput('target-repo-owner', targetRepoOwner) - core.setOutput('target-repo-name', targetRepoName) - - const sleep = async (milliseconds) => { - return new Promise(resolve => setTimeout(resolve, milliseconds)) - } - - const getRefs = async (repository) => { - let attemptCounter = 1 - for (;;) { - try { - const [owner, repo] = repository.split('/') - const { data } = await github.rest.git.listMatchingRefs({ - owner, - repo, - // We want to match `maint-*` as well as `[a-z][a-z]/*` - // sadly, this is not possible via GitHub's REST API, - // hence we do it below via the `filter()` call. - ref: 'heads/' - }) - return data - .filter(e => e.ref.match(/^refs\/heads\/(maint-\d|[a-z][a-z]\/)/)) - .sort((a, b) => a.ref.localeCompare(b.ref)) - } catch (e) { - if (e?.status !== 502) throw e - } - - if (++attemptCounter > 10) throw new Error('Giving up listing refs after 10 attempts') - - const seconds = attemptCounter * attemptCounter + 15 * Math.random() - core.info(`Encountered a Server Error; retrying in ${seconds} seconds`) - await sleep(1000 * seconds) - } - } - - const sourceRefs = await getRefs(process.env.SOURCE_REPOSITORY) - const targetRefs = await getRefs(process.env.TARGET_REPOSITORY) - - const refspecs = [] - const toFetch = new Set() - for (let i = 0, j = 0; i < sourceRefs.length || j < targetRefs.length; ) { - const compare = i >= sourceRefs.length - ? +1 - : j >= targetRefs.length - ? -1 - : sourceRefs[i].ref.localeCompare(targetRefs[j].ref) - if (compare > 0) { - // no source ref => delete target ref - refspecs.push(`:${targetRefs[j].ref}`) - j++ - } else if (compare < 0) { - // no corresponding target ref yet => push source ref (new) - const sha = sourceRefs[i].object.sha - toFetch.add(sha) - refspecs.push(`${sha}:${sourceRefs[i].ref}`) - i++ - } else { - // the sourceRef's name matches the targetRef's - if (sourceRefs[i].object.sha !== targetRefs[j].object.sha) { - // target ref needs updating - const sha = sourceRefs[i].object.sha - toFetch.add(sha) - refspecs.push(`+${sha}:${sourceRefs[i].ref}`) - } - i++ - j++ - } - } - - core.setOutput('refspec', refspecs.join(' ')) - targetRefs.forEach((e) => toFetch.delete(e.object.sha)) - core.setOutput('to-fetch', [...toFetch].join(' ')) - - name: obtain installation token - if: steps.check.outputs.refspec != '' - uses: actions/create-github-app-token@v2 - id: token - with: - app-id: ${{ secrets.GITGITGADGET_GITHUB_APP_ID }} - private-key: ${{ secrets.GITGITGADGET_GITHUB_APP_PRIVATE_KEY }} - owner: ${{ steps.check.outputs.target-repo-owner }} - repositories: ${{ steps.check.outputs.target-repo-name }} - - name: set authorization header - if: steps.check.outputs.refspec != '' - uses: actions/github-script@v7 - id: auth - with: - script: | - // Sadly, `git push` does not work with 'Authorization: Bearer ', therefore - // we have to use the `Basic` variant - const auth = Buffer.from('PAT:${{ steps.token.outputs.token }}').toString('base64') - core.setSecret(auth) - core.setOutput('header', `Authorization: Basic ${auth}`) - - name: sync - if: steps.check.outputs.refspec != '' - shell: bash - run: | - set -ex - git init --bare - - git remote add source "${{ github.server_url }}/$SOURCE_REPOSITORY" - # pretend to be a partial clone - git config remote.source.promisor true - git config remote.source.partialCloneFilter blob:none - - # fetch some commits - printf '%s' '${{ steps.check.outputs.to-fetch }}' | - xargs -d ' ' -r git fetch --depth 10000 source - rm -f .git/shallow - - # push the commits - printf '%s' '${{ steps.check.outputs.refspec }}' | - xargs -d ' ' -r git -c http.extraHeader='${{ steps.auth.outputs.header }}' \ - push "${{ github.server_url }}/$TARGET_REPOSITORY" diff --git a/.github/workflows/sync-git-mailing-list-mirror.yml b/.github/workflows/sync-mailing-list-mirror.yml similarity index 92% rename from .github/workflows/sync-git-mailing-list-mirror.yml rename to .github/workflows/sync-mailing-list-mirror.yml index 89d4bd6..b08b856 100644 --- a/.github/workflows/sync-git-mailing-list-mirror.yml +++ b/.github/workflows/sync-mailing-list-mirror.yml @@ -1,9 +1,9 @@ -name: sync-git-mailing-list-mirror +name: sync-mailing-list-mirror on: workflow_dispatch: schedule: - - cron: "2,7,12,17,22,27,32,37,42,47,52,57 * * * *" + - cron: "*/5 * * * *" env: LORE_EPOCH: 1 # also adjust SOURCE_REPOSITORY @@ -11,11 +11,11 @@ env: TARGET_GITHUB_REPOSITORY: gitgitgadget/git-mailing-list-mirror concurrency: - group: sync-git-mailing-list-mirror + group: sync-mailing-list-mirror cancel-in-progress: true jobs: - sync-git-mailing-list-mirror: + sync-mailing-list-mirror: runs-on: ubuntu-latest permissions: contents: write diff --git a/.github/workflows/sync-git-gui.yml b/.github/workflows/sync-upstream-branches.yml similarity index 69% rename from .github/workflows/sync-git-gui.yml rename to .github/workflows/sync-upstream-branches.yml index 5f49ac5..f1c39de 100644 --- a/.github/workflows/sync-git-gui.yml +++ b/.github/workflows/sync-upstream-branches.yml @@ -1,15 +1,10 @@ -name: sync-git-gui-branches +name: sync-upstream-branches on: schedule: - cron: '31 22 * * *' workflow_dispatch: -env: - SOURCE_REPOSITORY: j6t/git-gui - TARGET_REPOSITORY: gitgitgadget/git - TARGET_REF_NAMESPACE: git-gui/ - # We want to limit queuing to a single workflow run i.e. if there is already # an active workflow run and a queued one, queue another one canceling the # already queued one. @@ -17,15 +12,30 @@ concurrency: group: ${{ github.workflow }} jobs: - sync-git-gui-branches: + sync-upstream-branches: runs-on: ubuntu-latest + strategy: + matrix: + spec: + - sourceRepo: j6t/git-gui + targetRepo: gitgitgadget/git + targetRefNamespace: git-gui/ + - sourceRepo: gitster/git + targetRepo: gitgitgadget/git + sourceRefRegex: "^refs/heads/(maint-\\d|[a-z][a-z]/)" + steps: - name: check which refs need to be synchronized uses: actions/github-script@v7 id: check with: script: | - const [targetRepoOwner, targetRepoName] = process.env.TARGET_REPOSITORY.split('/') + const sourceRepo = ${{ toJSON(matrix.spec.sourceRepo) }} + const sourceRefRegexp = ((p) => p ? new RegExp(p) : null)(${{ toJSON(matrix.spec.sourceRefRegex) }}) + const targetRepo = ${{ toJSON(matrix.spec.targetRepo) }} + const targetRefNamespace = ${{ toJSON(matrix.spec.targetRefNamespace) }} || '' + + const [targetRepoOwner, targetRepoName] = targetRepo.split('/') core.setOutput('target-repo-owner', targetRepoOwner) core.setOutput('target-repo-name', targetRepoName) @@ -38,27 +48,26 @@ jobs: for (;;) { try { const [owner, repo] = repository.split('/') - let { data } = await github.rest.git.listMatchingRefs({ - owner, - repo, - ref: 'heads/' - }) - - data = data.filter(e => { - if (!e.ref.startsWith('refs/heads/')) return false - e.name = e.ref.slice(11) - return true - }) - - if (stripRefsPrefix) { - data = data.filter(e => { - if (!e.name.startsWith(stripRefsPrefix)) return false - e.name = e.name.slice(stripRefsPrefix.length) + return ( + await github.rest.git.listMatchingRefs({ + owner, + repo, + // We cannot match `source-ref-regex` as freely as we + // want with GitHub's REST API, hence we do it below via + // the `filter()` call. + ref: 'heads/' + }) + ).data + .filter((e) => { + if (sourceRefRegexp && !sourceRefRegexp.test(e.ref)) return false + if (!e.ref.startsWith('refs/heads/')) return false + e.name = e.ref.slice(11) + if (stripRefsPrefix) { + if (!e.name.startsWith(stripRefsPrefix)) return false + e.name = e.name.slice(stripRefsPrefix.length) + } return true }) - } - - return data .sort((a, b) => a.ref.localeCompare(b.ref)) } catch (e) { if (e?.status !== 502) throw e @@ -72,10 +81,10 @@ jobs: } } - const sourceRefs = await getRefs(process.env.SOURCE_REPOSITORY) - const targetRefs = await getRefs(process.env.TARGET_REPOSITORY, process.env.TARGET_REF_NAMESPACE) + const sourceRefs = await getRefs(sourceRepo) + const targetRefs = await getRefs(targetRepo, targetRefNamespace) - const targetPrefix = `refs/heads/${process.env.TARGET_REF_NAMESPACE}` + const targetPrefix = `refs/heads/${targetRefNamespace}` const refspecs = [] const toFetch = new Set() @@ -138,7 +147,7 @@ jobs: set -ex git init --bare - git remote add source "${{ github.server_url }}/$SOURCE_REPOSITORY" + git remote add source '${{ github.server_url }}/${{ matrix.spec.sourceRepo }}' # pretend to be a partial clone git config remote.source.promisor true git config remote.source.partialCloneFilter blob:none @@ -151,4 +160,4 @@ jobs: # push the commits printf '%s' '${{ steps.check.outputs.refspec }}' | xargs -d ' ' -r git -c http.extraHeader='${{ steps.auth.outputs.header }}' \ - push "${{ github.server_url }}/$TARGET_REPOSITORY" + push '${{ github.server_url }}/${{ matrix.spec.targetRepo }}' diff --git a/README.md b/README.md index ac8a925..28d4ef6 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,6 @@ This repository contains GitHub workflows, i.e. automated tasks, that keep [GitG ## Keeping the mirror of the Git mailing list up to date -The `sync-git-mailing-list.yml` workflow keeps the mirror at https://github.com/gitgitgadget/git-mailing-list of the Git mailing list mirror at https://lore.kernel.org/git up to date. Since that mirror chunks the archive by epochs, this mirror fetches each epoch into its own branch: the oldest epoch into `lore-0`, the next one into `lore-1`, etc. +The `sync-mailing-list-mirror.yml` workflow keeps the mirror at https://github.com/gitgitgadget/git-mailing-list of the Git mailing list mirror at https://lore.kernel.org/git up to date. Since that mirror chunks the archive by epochs, this mirror fetches each epoch into its own branch: the oldest epoch into `lore-0`, the next one into `lore-1`, etc. Previously, this workflow lived in the `git-mailing-list` repository in the `sync` branch, which was the default branch because scheduled workflows _must_ live in the default branch.