Transition from TravisCI to GitHub CI (#7444)

Convert the continuous integration process to use GitHub's internal CI cloud. Allows us to run up to 20 jobs in parallel, speeding up CI immensely. Keep a short Travis-CI run, too, just for sanity and backup. Uses new keys and secret when publishing a release to esp8266.github.io.
esp8266 · Jul 15, 2020 · 3bd2458 · 3bd2458
1 parent f809131
commit 3bd2458
Show file tree

Hide file tree

Showing 11 changed files with 442 additions and 173 deletions.
diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml
@@ -0,0 +1,259 @@
+# Run whenever a PR is generated or updated.
+
+# Most jobs check out the code, ensure Python3 is installed, and for build
+# tests the ESP8266 toolchain is cached when possible to speed up execution.
+
+name: ESP8266 Arduino CI
+
+on:
+  pull_request:
+
+
+jobs:
+
+# Run 8 parallel jobs for the default build of all examples.
+  build-linux:
+    name: Build ${{ matrix.chunk }}
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        chunk: [0, 1, 2, 3, 4, 5, 6, 7]
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        submodules: true
+    - uses: actions/setup-python@v2
+      with:
+        python-version: '3.x'
+    - name: Cache Linux toolchain
+      id: cache-linux
+      uses: actions/cache@v2
+      with:
+        path: ./tools/dist
+        key: key-linux-toolchain
+    - name: Build Sketches
+      env:
+        TRAVIS_BUILD_DIR: ${{ github.workspace }}
+        TRAVIS_TAG: ${{ github.ref }}
+        BUILD_PARITY: custom
+        mod: 8
+        rem: ${{ matrix.chunk }}
+      run: |
+        bash ./tests/build.sh
+
+
+# Cover the debug and IPv6 cases by enabling both and running 8 parallel jobs
+# over all example code.
+  build-debug-ipv6:
+    name: Debug IPv6 ${{ matrix.chunk }}
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        chunk: [0, 1, 2, 3, 4, 5, 6, 7]
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        submodules: true
+    - uses: actions/setup-python@v2
+      with:
+        python-version: '3.x'
+    - name: Cache Linux toolchain
+      id: cache-linux
+      uses: actions/cache@v2
+      with:
+        path: ./tools/dist
+        key: key-linux-toolchain
+    - name: Build Sketches
+      env:
+        TRAVIS_BUILD_DIR: ${{ github.workspace }}
+        TRAVIS_TAG: ${{ github.ref }}
+        BUILD_PARITY: custom
+        mod: 8
+        rem: ${{ matrix.chunk }}
+      run: |
+        bash ./tests/debug6.sh
+
+
+# Single build under Windows to ensure the Win toolchain is good.
+  build-windows:
+    name: Windows
+    runs-on: windows-latest
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        submodules: true
+    - uses: actions/setup-python@v2
+      with:
+        python-version: '3.x'
+    - name: Cache Windows toolchain
+      id: cache-windows
+      uses: actions/cache@v2
+      with:
+        path: ./tools/dist
+        key: key-windows-toolchain
+    - name: Build Sketch
+      env:
+        TRAVIS_BUILD_DIR: ${{ github.workspace }}
+        TRAVIS_TAG: ${{ github.ref }}
+        WINDOWS: 1
+        BUILD_PARITY: custom
+        mod: 500
+        rem: 1
+      run: |
+        # Windows has python3 already installed, but it's called "python".
+        # Copy python.exe to the proper name so scripts "just work".
+        copy (get-command python).source (get-command python).source.Replace("python.exe", "python3.exe")
+        bash ./tests/build.sh
+
+
+# Single build under macOS to ensure the Mac toolchain is good.
+  build-mac:
+    name: Mac
+    runs-on: macOS-latest
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        submodules: true
+    - uses: actions/setup-python@v2
+      with:
+        python-version: '3.x'
+    - name: Cache Mac toolchain
+      id: cache-mac
+      uses: actions/cache@v2
+      with:
+        path: ./tools/dist
+        key: key-mac-toolchain
+    - name: Build Sketch
+      env:
+        TRAVIS_BUILD_DIR: ${{ github.workspace }}
+        TRAVIS_TAG: ${{ github.ref }}
+        MACOSX: 1
+        BUILD_PARITY: custom
+        mod: 500
+        rem: 1
+      run: |
+        bash ./tests/build.sh
+
+
+# Run a few Platform.IO jobs (not full suite) to check PIO integration.
+  build-pio:
+    name: Build Platform.IO
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        submodules: true
+    - uses: actions/setup-python@v2
+      with:
+        python-version: '3.x'
+    - name: Build subset on Platform.IO
+      env:
+        TRAVIS_BUILD_DIR: ${{ github.workspace }}
+        TRAVIS_TAG: ${{ github.ref }}
+        BUILD_PARITY: custom
+        mod: 42 # Picked at random to give 4-5 builds and exit.
+        rem: 13
+      run: |
+        sudo apt-get install python3-pip python3-setuptools
+        PATH=/home/runner/.local/bin:$PATH bash ./tests/platformio.sh
+
+
+# Run host test suite under valgrind for runtime checking of code.
+  host-tests:
+    name: Host tests
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        submodules: true
+    - uses: actions/setup-python@v2
+      with:
+        python-version: '3.x'
+    - name: Run host tests
+      env:
+        TRAVIS_BUILD_DIR: ${{ github.workspace }}
+        TRAVIS_TAG: ${{ github.ref }}
+      run: |
+        sudo apt-get install valgrind lcov
+        bash ./tests/ci/host_test.sh
+
+
+# Ensure Sphinx can build the documentation properly.
+  documentation:
+    name: Documentation
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        submodules: true
+    - uses: actions/setup-python@v2
+      with:
+        python-version: '3.x'
+    - name: Build documentation
+      env:
+        TRAVIS_BUILD_DIR: ${{ github.workspace }}
+        TRAVIS_TAG: ${{ github.ref }}
+      run: |
+        sudo apt-get install python3-pip python3-setuptools
+        # GitHub CI installs pip3 and setuptools outside the path.
+        # Update the path to include them and run.
+        PATH=/home/runner/.local/bin:$PATH pip3 install --user -r doc/requirements.txt
+        PATH=/home/runner/.local/bin:$PATH bash ./tests/ci/build_docs.sh
+
+
+# Standard Arduino formatting in all the examples
+  style-check:
+    name: Style and formatting
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        submodules: true
+    - uses: actions/setup-python@v2
+      with:
+        python-version: '3.x'
+    - name: Style check
+      env:
+        TRAVIS_BUILD_DIR: ${{ github.workspace }}
+        TRAVIS_TAG: ${{ github.ref }}
+      run: |
+          sudo apt-get install astyle
+          bash ./tests/ci/style_check.sh
+
+
+# Quick test that the mocking builds succeed
+  mock-check:
+    name: Mock trivial test
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        submodules: true
+    - uses: actions/setup-python@v2
+      with:
+        python-version: '3.x'
+    - name: Mock build
+      env:
+        TRAVIS_BUILD_DIR: ${{ github.workspace }}
+        TRAVIS_TAG: ${{ github.ref }}
+      run: |
+          bash ./tests/buildm.sh
+
+
+# Ensure no manual edits to boards.txt
+  boards-check:
+    name: Boards.txt check
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        submodules: true
+    - uses: actions/setup-python@v2
+      with:
+        python-version: '3.x'
+    - name: Boards.txt diff
+      env:
+        TRAVIS_BUILD_DIR: ${{ github.workspace }}
+        TRAVIS_TAG: ${{ github.ref }}
+      run: |
+          bash ./tests/ci/build_boards.sh
diff --git a/.github/workflows/release-to-publish.yml b/.github/workflows/release-to-publish.yml
@@ -0,0 +1,56 @@
+# Whenever a release is published from a draft, this will update the
+# master Arduino JSON file to add its new entry.
+
+# We keep the master JSON file in another repo, so we need to use a pre-set
+# Deployment SSH key to be able to push a change to the repo.
+
+#### Steps to follow when you need to make a new SSH key for upload (not
+#### normally needed!)
+
+# Generate a new SSH key private/public pair
+
+# ssh-keygen -t rsa -b 4096 -C "your@email.com" -f ./deploy_rsa
+
+# Upload deploy_rsa.pub to the *ESP8266.GITHUB.IO* repo as a deployment key
+
+# Convert the private key to base64 (to remove line breaks and allow easier
+# usage in the script as an environment variable)
+
+# base64.exe -w 0 < deploy_rsa > deploy_rsa.b64
+
+# Copy the contents of the .b64 file to the clipboard, make a new GitHub
+# secret in the ESP8266/Arduino repo called "GHCI_DEPLOY_KEY" and paste
+# the B64 code into the variable.
+
+name: ESP8266 Arduino Release Publisher
+
+on:
+  release:
+    types: [published]
+
+jobs:
+  package:
+    name: Update master JSON file
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        submodules: true
+    - uses: actions/setup-python@v2
+      with:
+        python-version: '3.x'
+    - name: Set GIT tag name
+      run: |
+        echo "::set-env name=TRAVIS_TAG::$(git describe --exact-match --tags)"
+    - name: Deploy updated JSON
+      env:
+        TRAVIS_BUILD_DIR: ${{ github.workspace }}
+        BUILD_TYPE: package
+        CI_GITHUB_API_KEY: ${{ secrets.GITHUB_TOKEN }}
+        GHCI_DEPLOY_KEY: ${{ secrets.GHCI_DEPLOY_KEY }}
+      run: |
+           bash ./tests/ci/build_package.sh
+           # Only the regenerated JSON file will be used, but it's simpler
+           # than looking for it in a GH release.
+           bash ./package/deploy_package_index.sh
+
diff --git a/.github/workflows/tag-to-draft-release.yml b/.github/workflows/tag-to-draft-release.yml
@@ -0,0 +1,40 @@
+# Whenever a tag of the form #.xxxx is pushed against master, generate a
+# draft release and upload the ZIP and JSON file to it.  Maintainers then
+# will manually add the changelist and publish it.
+
+name: ESP8266 Arduino Draft Release
+
+on:
+  push:
+    tags:
+      # Run for tags of the x.x.x* form (i.e. 3.0.0, 3.0.0-beta, etc.).
+      - '[0-9]+.[0-9]+.[0-9]+*'
+
+jobs:
+  package:
+    name: Package
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        submodules: true
+    - uses: actions/setup-python@v2
+      with:
+        python-version: '3.x'
+    - name: Set GIT tag name
+      run: |
+        # Sets an environment variable used in the next steps
+        echo "::set-env name=TRAVIS_TAG::$(git describe --exact-match --tags)"
+    - name: Build package JSON
+      env:
+        TRAVIS_BUILD_DIR: ${{ github.workspace }}
+        BUILD_TYPE: package
+        CI_GITHUB_API_KEY: ${{ secrets.GITHUB_TOKEN }}
+      run: |
+           bash ./tests/ci/build_package.sh
+           pip3 install PyGithub
+           # Create a draft release and upload the ZIP and JSON files.
+           # This draft is not visible to normal users and needs to be
+           # updated manually with release notes and published from the
+           # GitHub web interface.
+           python3 ./package/upload_release.py --user "$GITHUB_ACTOR" --repo "$GITHUB_REPOSITORY" --token "$CI_GITHUB_API_KEY" --tag "$TRAVIS_TAG" --name "Release $TRAVIS_TAG" --msg "Update the draft with release notes before publishing." package/versions/*/*.zip package/versions/*/package_esp8266com_index.json