MakeTrie #209
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: MakeTrie | |
| on: | |
| # docs.github.com/en/actions/learn-github-actions/events-that-trigger-workflows#schedule | |
| schedule: | |
| # at 21:53 on 1st, 8th, 16th, 24th of every month | |
| - cron: '53 21 1,8,16,24 * *' | |
| workflow_dispatch: | |
| inputs: | |
| publish: | |
| description: 'publish blocklists?' | |
| required: true | |
| default: 'test' | |
| type: choice | |
| options: | |
| # "yes" / "no" convert to bools, hence use "yeah" / "nope" instead. | |
| # bools are bools when accessed with input-context, but are | |
| # stringified when accessed with github.events context. And so: | |
| # for the sake of everyone's sanity, stick to non-boolean inputs. | |
| # ref: docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#providing-inputs | |
| - yeah | |
| - nope | |
| - test | |
| env: | |
| INDIR: "blocklistfiles" | |
| OUTDIR: "result" | |
| S3DIR: "blocklists" | |
| S3_TEST_DIR: "testblocklists" | |
| BLCONFIG: "config.json" | |
| PREFER_S3_OVER_R2: "false" | |
| jobs: | |
| version: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 1 | |
| # docs.github.com/en/actions/learn-github-actions/contexts#example-usage-of-the-needs-context | |
| outputs: | |
| epoch: ${{ steps.v.outputs.epoch }} | |
| steps: | |
| - name: Gen version | |
| id: v | |
| run: | | |
| # stackoverflow.com/a/69400542 | |
| v=$(date --utc +"%s.%3N") | |
| echo "epoch=${v}" >> $GITHUB_OUTPUT | |
| echo "::notice::Version ${v} / ${SCHED}" | |
| shell: bash | |
| env: | |
| SCHED: ${{ github.event.schedule }} | |
| make: | |
| needs: version | |
| strategy: | |
| matrix: | |
| codec: ["u8", "u6"] | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 100 | |
| steps: | |
| - name: 📃 Checkout repo | |
| uses: actions/checkout@v3.3.0 | |
| - name: 🐉 Setup python | |
| uses: actions/setup-python@v4.5.0 | |
| with: | |
| python-version: '3.x' | |
| - name: ⭕ Setup node | |
| uses: actions/setup-node@v3.6.0 | |
| with: | |
| node-version: '19' | |
| - name: ⏳ Make swap | |
| run: | | |
| free -hm | |
| df -h | |
| # make swap: community.fly.io/t/6782/10 | |
| sudo fallocate -l 16G /tmp/swapfile | |
| # unix.stackexchange.com/a/297155 | |
| sudo chown root:root /tmp/swapfile | |
| sudo mkswap /tmp/swapfile | |
| # stackoverflow.com/a/550808 | |
| sudo cat /proc/sys/vm/swappiness | |
| echo "20" | sudo tee /proc/sys/vm/swappiness > /dev/null | |
| # github.com/pierotofy/set-swap-space/blob/master/action.yml | |
| sudo swapon /tmp/swapfile | |
| free -hm | |
| df -h | |
| shell: bash | |
| - name: 🎨 Setup | |
| id: setup | |
| run: | | |
| # install deps | |
| python -m pip install aiohttp | |
| npm install | |
| # determine config | |
| # all output vars are strings | |
| # docs.github.com/en/actions/learn-github-actions/contexts#steps-context | |
| echo "upload=y" >> $GITHUB_OUTPUT | |
| if [ "$PUBLISH_BLOCKLISTS" = "test" ]; then | |
| echo "S3DIR=$S3_TEST_DIR" >> $GITHUB_ENV | |
| elif [ "$PUBLISH_BLOCKLISTS" = "nope" ]; then | |
| echo "S3DIR=" >> $GITHUB_ENV | |
| # do not upload | |
| echo "upload=n" >> $GITHUB_OUTPUT | |
| fi | |
| echo "UNIX_EPOCH_SEC=${V}" >> $GITHUB_ENV | |
| echo "::notice::Building ${GIT_REF}/${COMMIT_SHA} @ ${V}" | |
| shell: bash | |
| env: | |
| COMMIT_SHA: ${{ github.sha }} | |
| GIT_REF: ${{ github.ref }} | |
| V: ${{ needs.version.outputs.epoch }} | |
| PUBLISH_BLOCKLISTS: ${{ inputs.publish }} | |
| - name: 📥 Download blocklists | |
| run: | | |
| python download.py | |
| env: | |
| BLCONFIG: ${{ env.BLCONFIG }} | |
| - name: 🌲 Make compact trie | |
| run: | | |
| mkdir -p "$OUTDIR" | |
| # 32768/1024 => 32G RAM | |
| node --max-old-space-size=32768 --expose-gc ./src/build.js | |
| env: | |
| BLCONFIG: ${{ env.BLCONFIG }} | |
| OUTDIR: ${{ env.OUTDIR }} | |
| INDIR: ${{ env.INDIR }} | |
| UNIX_EPOCH_SEC: ${{ env.UNIX_EPOCH_SEC }} | |
| CODEC: ${{ matrix.codec }} | |
| - name: 🪄 Show artifacts | |
| run: | | |
| # note: split now happens in serverless-dns/trie:trie.js | |
| # creates td00.txt, td01.txt, ... , td98.txt, td99.txt, td100.txt, ... | |
| # cd "$OUTDIR" && split --bytes=30M -d --additional-suffix=.txt td.txt td | |
| cd "$OUTDIR" && ls -lhtr | |
| env: | |
| OUTDIR: ${{ env.OUTDIR }} | |
| - name: 🏆 Upload? | |
| # stackoverflow.com/a/68317611 | |
| # github.com/actions/runner/issues/866 | |
| if: ${{ steps.setup.outputs.upload == 'y' }} | |
| run: | | |
| node ./src/upload.js | |
| env: | |
| AWS_ACCESS_KEY: ${{ secrets.AWS_ACCESS_KEY }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| AWS_BUCKET_NAME: ${{ secrets.AWS_BUCKET_NAME }} | |
| CF_ACCOUNT_ID: ${{ secrets.CF_ACCOUNT_ID }} | |
| PREFER_S3_OVER_R2: ${{ env.PREFER_S3_OVER_R2 }} | |
| UNIX_EPOCH_SEC: ${{ env.UNIX_EPOCH_SEC }} | |
| CODEC: ${{ matrix.codec }} | |
| S3DIR: ${{ env.S3DIR }} | |
| OUTDIR: ${{ env.OUTDIR }} |