dendrite.yml

name: Dendrite

on:
  push:
    branches:
      - main
    paths:
      - '**.go'                 # only execute on changes to go files
      - 'go.sum'                # or dependency updates
      - '.github/workflows/**'  # or workflow changes
  pull_request:
    paths:
      - '**.go'
      - 'go.sum'                # or dependency updates
      - '.github/workflows/**'
  release:
    types: [published]
  workflow_dispatch:

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

jobs:
  wasm:
    name: WASM build test
    timeout-minutes: 5
    runs-on: ubuntu-latest
    if: ${{ false }} # disable for now
    steps:
      - uses: actions/checkout@v4

      - name: Install Go
        uses: actions/setup-go@v4
        with:
          go-version-file: 'go.mod'
          cache: true

      - name: Install Node
        uses: actions/setup-node@v2
        with:
          node-version: 14

      - uses: actions/cache@v4
        with:
          path: ~/.npm
          key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
          restore-keys: |
            ${{ runner.os }}-node-

      - name: Reconfigure Git to use HTTPS auth for repo packages
        run: >
          git config --global url."https://github.com/".insteadOf
          ssh://git@github.com/

      - name: Install test dependencies
        working-directory: ./test/wasm
        run: npm ci

      - name: Test
        run: ./test-dendritejs.sh

  # Run golangci-lint
  lint:
    timeout-minutes: 5
    name: Linting
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Install libolm
        run: sudo apt-get install libolm-dev libolm3
      - name: Install Go
        uses: actions/setup-go@v4
        with:
          go-version-file: 'go.mod'
      - name: golangci-lint
        uses: golangci/golangci-lint-action@v3

  # run go test with different go versions
  test:
    timeout-minutes: 10
    name: Unit tests
    runs-on: ubuntu-latest
    # Service containers to run with `container-job`
    services:
      # Label used to access the service container
      postgres:
        # Docker Hub image
        image: postgres:13-alpine
        # Provide the password for postgres
        env:
          POSTGRES_USER: postgres
          POSTGRES_PASSWORD: postgres
          POSTGRES_DB: dendrite
        ports:
          # Maps tcp port 5432 on service container to the host
          - 5432:5432
        # Set health checks to wait until postgres has started
        options: >-
          --health-cmd pg_isready
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5
    steps:
      - uses: actions/checkout@v4
      - name: Install libolm
        run: sudo apt-get install libolm-dev libolm3
      - name: Setup go
        uses: actions/setup-go@v4
        with:
          go-version-file: 'go.mod'
      - uses: actions/cache@v4
        # manually set up caches, as they otherwise clash with different steps using setup-go with cache=true
        with:
          path: |
            ~/.cache/go-build
            ~/go/pkg/mod
          key: ${{ runner.os }}-go-stable-unit-${{ hashFiles('**/go.sum') }}
          restore-keys: |
            ${{ runner.os }}-go-stable-unit-
      - name: Set up gotestfmt
        uses: gotesttools/gotestfmt-action@v2
        with:
          # Optional: pass GITHUB_TOKEN to avoid rate limiting.
          token: ${{ secrets.GITHUB_TOKEN }}
      - run: go test -json -v ./... 2>&1 | gotestfmt -hide all
        env:
          POSTGRES_HOST: localhost
          POSTGRES_USER: postgres
          POSTGRES_PASSWORD: postgres
          POSTGRES_DB: dendrite

  # build Dendrite for linux with different architectures and go versions
  build:
    name: Build for Linux
    timeout-minutes: 10
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        goos: ["linux"]
        goarch: ["amd64", "386"]
    steps:
      - uses: actions/checkout@v4
      - name: Setup go
        uses: actions/setup-go@v4
        with:
          go-version-file: 'go.mod'
      - uses: actions/cache@v4
        with:
          path: |
            ~/.cache/go-build
            ~/go/pkg/mod
          key: ${{ runner.os }}-go-stable-${{ matrix.goos }}-${{ matrix.goarch }}-${{ hashFiles('**/go.sum') }}
          restore-keys: |
            key: ${{ runner.os }}-go-stable-${{ matrix.goos }}-${{ matrix.goarch }}-
      - name: Install dependencies x86
        if: ${{ matrix.goarch == '386' }}
        run: sudo apt update && sudo apt-get install -y gcc-multilib
      - env:
          GOOS: ${{ matrix.goos }}
          GOARCH: ${{ matrix.goarch }}
          CGO_ENABLED: 1
          CGO_CFLAGS: -fno-stack-protector
        run: go build -trimpath -v -o "bin/" ./cmd/...

  # build for Windows 64-bit
  build_windows:
    name: Build for Windows
    timeout-minutes: 10
    runs-on: ubuntu-latest
    strategy:
      matrix:
        goos: ["windows"]
        goarch: ["amd64"]
    steps:
      - uses: actions/checkout@v4
      - name: Setup Go
        uses: actions/setup-go@v4
        with:
          go-version-file: 'go.mod'
      - uses: actions/cache@v4
        with:
          path: |
            ~/.cache/go-build
            ~/go/pkg/mod
          key: ${{ runner.os }}-go-stable-${{ matrix.goos }}-${{ matrix.goarch }}-${{ hashFiles('**/go.sum') }}
          restore-keys: |
            key: ${{ runner.os }}-go-stable-${{ matrix.goos }}-${{ matrix.goarch }}-
      - name: Install dependencies
        run: sudo apt update && sudo apt install -y gcc-mingw-w64-x86-64 # install required gcc
      - env:
          GOOS: ${{ matrix.goos }}
          GOARCH: ${{ matrix.goarch }}
          CGO_ENABLED: 1
          CC: "/usr/bin/x86_64-w64-mingw32-gcc"
        run: go build -trimpath -v -o "bin/" ./cmd/...

  # Dummy step to gate other tests on without repeating the whole list
  initial-tests-done:
    name: Initial tests passed
    needs: [lint, test, build, build_windows]
    runs-on: ubuntu-latest
    if: ${{ !cancelled() }} # Run this even if prior jobs were skipped
    steps:
      - name: Check initial tests passed
        uses: re-actors/alls-green@release/v1
        with:
          jobs: ${{ toJSON(needs) }}

  # run go test with different go versions
  integration:
    timeout-minutes: 20
    needs: initial-tests-done
    name: Integration tests
    runs-on: ubuntu-latest
    # Service containers to run with `container-job`
    services:
      # Label used to access the service container
      postgres:
        # Docker Hub image
        image: postgres:13-alpine
        # Provide the password for postgres
        env:
          POSTGRES_USER: postgres
          POSTGRES_PASSWORD: postgres
          POSTGRES_DB: dendrite
        ports:
          # Maps tcp port 5432 on service container to the host
          - 5432:5432
        # Set health checks to wait until postgres has started
        options: >-
          --health-cmd pg_isready
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5
    steps:
      - uses: actions/checkout@v4
      - name: Install libolm
        run: sudo apt-get install libolm-dev libolm3
      - name: Setup go
        uses: actions/setup-go@v4
        with:
          go-version-file: 'go.mod'
      - name: Set up gotestfmt
        uses: gotesttools/gotestfmt-action@v2
        with:
          # Optional: pass GITHUB_TOKEN to avoid rate limiting.
          token: ${{ secrets.GITHUB_TOKEN }}
      - uses: actions/cache@v4
        with:
          path: |
            ~/.cache/go-build
            ~/go/pkg/mod
          key: ${{ runner.os }}-go-stable-test-race-${{ hashFiles('**/go.sum') }}
          restore-keys: |
            ${{ runner.os }}-go-stable-test-race-
      - run: go test -race -json -v -coverpkg=./... -coverprofile=cover.out $(go list ./... | grep -v /cmd/dendrite*) 2>&1 | gotestfmt -hide all
        env:
          POSTGRES_HOST: localhost
          POSTGRES_USER: postgres
          POSTGRES_PASSWORD: postgres
          POSTGRES_DB: dendrite
      - name: Upload coverage to Codecov
        uses: codecov/codecov-action@v4
        with:
          flags: unittests
          fail_ci_if_error: true
          token: ${{ secrets.CODECOV_TOKEN }}

  # run database upgrade tests
  upgrade_test:
    name: Upgrade tests
    timeout-minutes: 20
    needs: initial-tests-done
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Setup go
        uses: actions/setup-go@v4
        with:
          go-version-file: 'go.mod'
          cache: true
      - uses: actions/cache@v4
        with:
          path: |
            ~/.cache/go-build
            ~/go/pkg/mod
          key: ${{ runner.os }}-go-upgrade-test-${{ hashFiles('**/go.sum') }}
          restore-keys: |
            ${{ runner.os }}-go-upgrade-test-
      - name: Docker version
        run: docker version
      - name: Build upgrade-tests
        run: go build ./cmd/dendrite-upgrade-tests
      - name: Test upgrade (PostgreSQL)
        run: ./dendrite-upgrade-tests --head .
      - name: Test upgrade (SQLite)
        run: ./dendrite-upgrade-tests --sqlite --head .

  # run database upgrade tests, skipping over one version
  upgrade_test_direct:
    name: Upgrade tests from HEAD-2
    timeout-minutes: 20
    needs: initial-tests-done
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Setup go
        uses: actions/setup-go@v4
        with:
          go-version-file: 'go.mod'
          cache: true
      - uses: actions/cache@v4
        with:
          path: |
            ~/.cache/go-build
            ~/go/pkg/mod
          key: ${{ runner.os }}-go-upgrade-direct-test-${{ hashFiles('**/go.sum') }}
          restore-keys: |
            ${{ runner.os }}-go-upgrade-direct-test-
      - name: Docker version
        run: docker version
      - name: Build upgrade-tests
        run: go build ./cmd/dendrite-upgrade-tests
      - name: Test upgrade (PostgreSQL)
        run: ./dendrite-upgrade-tests -direct -from HEAD-2 --head .
      - name: Test upgrade (SQLite)
        run: ./dendrite-upgrade-tests -direct -from HEAD-2 --head .

  # run Sytest in different variations
  sytest:
    timeout-minutes: 20
    needs: initial-tests-done
    name: "Sytest (${{ matrix.label }})"
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        include:
          - label: SQLite native

          - label: SQLite Cgo
            cgo: 1

          - label: PostgreSQL
            postgres: postgres

    container:
      image: matrixdotorg/sytest-dendrite
      volumes:
        - ${{ github.workspace }}:/src
        - /root/.cache/go-build:/github/home/.cache/go-build
        - /root/.cache/go-mod:/gopath/pkg/mod
      env:
        POSTGRES: ${{ matrix.postgres && 1}}
        SYTEST_BRANCH: ${{ github.head_ref }}
        CGO_ENABLED: ${{ matrix.cgo && 1 }}
    steps:
      - uses: actions/checkout@v4
      - uses: actions/cache@v4
        with:
          path: |
            ~/.cache/go-build
            /gopath/pkg/mod
          key: ${{ runner.os }}-go-sytest-${{ hashFiles('**/go.sum') }}
          restore-keys: |
            ${{ runner.os }}-go-sytest-
      - name: Run Sytest
        run: /bootstrap.sh dendrite
        working-directory: /src
      - name: Summarise results.tap
        if: ${{ always() }}
        run: /sytest/scripts/tap_to_gha.pl /logs/results.tap
      - name: Sytest List Maintenance
        if: ${{ always() }}
        run: /src/show-expected-fail-tests.sh /logs/results.tap /src/sytest-whitelist /src/sytest-blacklist
        continue-on-error: true # not fatal
      - name: Are We Synapse Yet?
        if: ${{ always() }}
        run: /src/are-we-synapse-yet.py /logs/results.tap -v
        continue-on-error: true # not fatal
      - name: Upload Sytest logs
        uses: actions/upload-artifact@v4
        if: ${{ always() }}
        with:
          name: Sytest Logs - ${{ job.status }} - (Dendrite, ${{ join(matrix.*, ', ') }})
          path: |
            /logs/results.tap
            /logs/**/*.log*

  # run Complement
  complement:
    name: "Complement (${{ matrix.label }})"
    timeout-minutes: 20
    needs: initial-tests-done
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        include:
          - label: SQLite native
            cgo: 0

          - label: SQLite Cgo
            cgo: 1

          - label: PostgreSQL
            postgres: Postgres
            cgo: 0
    steps:
      # Env vars are set file a file given by $GITHUB_PATH. We need both Go 1.17 and GOPATH on env to run Complement.
      # See https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#adding-a-system-path
      - name: "Set Go Version"
        run: |
          echo "$GOROOT_1_17_X64/bin" >> $GITHUB_PATH
          echo "~/go/bin" >> $GITHUB_PATH
      - name: "Install Complement Dependencies"
        # We don't need to install Go because it is included on the Ubuntu 20.04 image:
        # See https://github.com/actions/virtual-environments/blob/main/images/linux/Ubuntu2004-Readme.md specifically GOROOT_1_17_X64
        run: |
          sudo apt-get update && sudo apt-get install -y libolm3 libolm-dev
          go install github.com/gotesttools/gotestfmt/v2/cmd/gotestfmt@latest
      - name: Run actions/checkout@v4 for dendrite
        uses: actions/checkout@v4
        with:
          path: dendrite

      # Attempt to check out the same branch of Complement as the PR. If it
      # doesn't exist, fallback to main.
      - name: Checkout complement
        shell: bash
        run: |
          mkdir -p complement
          # Attempt to use the version of complement which best matches the current
          # build. Depending on whether this is a PR or release, etc. we need to
          # use different fallbacks.
          #
          # 1. First check if there's a similarly named branch (GITHUB_HEAD_REF
          #    for pull requests, otherwise GITHUB_REF).
          # 2. Attempt to use the base branch, e.g. when merging into release-vX.Y
          #    (GITHUB_BASE_REF for pull requests).
          # 3. Use the default complement branch ("master").
          for BRANCH_NAME in "$GITHUB_HEAD_REF" "$GITHUB_BASE_REF" "${GITHUB_REF#refs/heads/}" "master"; do
            # Skip empty branch names and merge commits.
            if [[ -z "$BRANCH_NAME" || $BRANCH_NAME =~ ^refs/pull/.* ]]; then
              continue
            fi
            (wget -O - "https://github.com/matrix-org/complement/archive/$BRANCH_NAME.tar.gz" | tar -xz --strip-components=1 -C complement) && break
          done
      # Build initial Dendrite image
      - run: docker build --build-arg=CGO=${{ matrix.cgo }} -t complement-dendrite:${{ matrix.postgres }}${{ matrix.cgo }} -f build/scripts/Complement${{ matrix.postgres }}.Dockerfile .
        working-directory: dendrite
        env:
          DOCKER_BUILDKIT: 1

      # Run Complement
      - run: |
          set -o pipefail &&
          go test -v -json -tags dendrite_blacklist ./tests ./tests/csapi 2>&1 | gotestfmt -hide all
        shell: bash
        name: Run Complement Tests
        env:
          COMPLEMENT_BASE_IMAGE: complement-dendrite:${{ matrix.postgres }}${{ matrix.cgo }}
          COMPLEMENT_SHARE_ENV_PREFIX: COMPLEMENT_DENDRITE_
        working-directory: complement

  integration-tests-done:
    name: Integration tests passed
    needs:
      [
        initial-tests-done,
        upgrade_test,
        upgrade_test_direct,
        sytest,
        complement,
        integration
      ]
    runs-on: ubuntu-latest
    if: ${{ !cancelled() }} # Run this even if prior jobs were skipped
    steps:
      - name: Check integration tests passed
        uses: re-actors/alls-green@release/v1
        with:
          jobs: ${{ toJSON(needs) }}

  update-docker-images:
    name: Update Docker images
    permissions:
      packages: write
      contents: read
      security-events: write # To upload Trivy sarif files
    if: github.repository == 'matrix-org/dendrite' && github.ref_name == 'main'
    needs: [integration-tests-done]
    uses: matrix-org/dendrite/.github/workflows/docker.yml@main
    secrets:
      DOCKER_TOKEN: ${{ secrets.DOCKER_TOKEN }}