development.yaml

name: Development
on:
  push:
    branches:
      - "**" # every branch
      - "!gh-pages" # exclude gh-pages branch
      - "!stage*" # exclude branches beginning with stage
    tags:
      - '\d+\.\d+\.\d+' # only semver tags
  pull_request:
    branches:
      - "**" # every branch
      - "!gh-pages" # exclude gh-pages branch
      - "!stage*" # exclude branches beginning with stage
jobs:
  build:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        include:
          - py_ver: "3.9"
            distro: debian
            image: djbase
    env:
      PY_VER: ${{matrix.py_ver}}
      DISTRO: ${{matrix.distro}}
      IMAGE: ${{matrix.image}}
      DOCKER_CLIENT_TIMEOUT: "120"
      COMPOSE_HTTP_TIMEOUT: "120"
    steps:
      - uses: actions/checkout@v4
      - name: Validate version and release notes
        run: |
          DJ_VERSION=$(grep -oP '\d+\.\d+\.\d+' datajoint/version.py)
          RELEASE_BODY=$(python -c \
            'print(open("./CHANGELOG.md").read().split("\n\n")[1].split("\n", 1)[1])' \
          )
          echo "DJ_VERSION=${DJ_VERSION}" >> $GITHUB_ENV
          echo "RELEASE_BODY<<EOF" >> $GITHUB_ENV
          echo "$RELEASE_BODY" >> $GITHUB_ENV
          echo "EOF" >> $GITHUB_ENV
      - name: Build pip artifacts
        run: |
          export HOST_UID=$(id -u)
          docker compose -f docker-compose-build.yaml up --exit-code-from app --build
          echo "DJ_VERSION=${DJ_VERSION}" >> $GITHUB_ENV
      - if: matrix.py_ver == '3.9' && matrix.distro == 'debian'
        name: Add pip artifacts
        uses: actions/upload-artifact@v3
        with:
          name: pip-datajoint-${{env.DJ_VERSION}}
          path: dist
          retention-days: 1
  test:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        py_ver: ["3.9"]
        mysql_ver: ["8.0", "5.7"]
        include:
          - py_ver: "3.11"
            mysql_ver: "8.0"
          - py_ver: "3.10"
            mysql_ver: "8.0"
          - py_ver: "3.8"
            mysql_ver: "5.7"
    steps:
      - uses: actions/checkout@v4
      - name: Set up Python ${{matrix.py_ver}}
        uses: actions/setup-python@v5
        with:
          python-version: ${{matrix.py_ver}}
      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install flake8 black
      - name: Run syntax tests
        run: flake8 datajoint --count --select=E9,F63,F7,F82 --show-source --statistics
      - name: Run primary tests
        env:
          PY_VER: ${{matrix.py_ver}}
          DJ_PASS: password
          MYSQL_VER: ${{matrix.mysql_ver}}
          DISTRO: alpine
          MINIO_VER: RELEASE.2021-09-03T03-56-13Z
          DOCKER_CLIENT_TIMEOUT: "120"
          COMPOSE_HTTP_TIMEOUT: "120"
        run: |
          export HOST_UID=$(id -u)
          docker compose -f LNX-docker-compose.yml up --build --exit-code-from app
  lint:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        py_ver: ["3.11"]
    steps:
      - uses: actions/checkout@v4
      - name: Set up Python ${{matrix.py_ver}}
        uses: actions/setup-python@v5
        with:
          python-version: ${{matrix.py_ver}}
      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install flake8 black==24.2.0
      - name: Run syntax tests
        run: flake8 datajoint --count --select=E9,F63,F7,F82 --show-source --statistics
      - name: Run style tests
        run: |
          flake8 --ignore=E203,E722,W503 datajoint \
                 --count --max-complexity=62 --max-line-length=127 --statistics \
                 --per-file-ignores='datajoint/diagram.py:C901'
          black --required-version '24.2.0' --check -v datajoint tests
  codespell:
    name: Check for spelling errors
    permissions:
      contents: read
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Codespell
        uses: codespell-project/actions-codespell@v2
  publish-docs:
    if: |
      github.event_name == 'push' &&
      startsWith(github.ref, 'refs/tags')
    needs: test
    runs-on: ubuntu-latest
    env:
      DOCKER_CLIENT_TIMEOUT: "120"
      COMPOSE_HTTP_TIMEOUT: "120"
    steps:
      - uses: actions/checkout@v4
      - name: Deploy docs
        run: |
          export MODE=BUILD
          export PACKAGE=datajoint
          export UPSTREAM_REPO=https://github.com/${GITHUB_REPOSITORY}.git
          export HOST_UID=$(id -u)
          docker compose -f docs/docker-compose.yaml up --exit-code-from docs --build
          git push origin gh-pages
  publish-release:
    if: |
      github.event_name == 'push' &&
      startsWith(github.ref, 'refs/tags')
    needs: test
    runs-on: ubuntu-latest
    strategy:
      matrix:
        include:
          - py_ver: "3.9"
            distro: debian
            image: djbase
    env:
      PY_VER: ${{matrix.py_ver}}
      DISTRO: ${{matrix.distro}}
      IMAGE: ${{matrix.image}}
      TWINE_USERNAME: ${{secrets.twine_username}}
      TWINE_PASSWORD: ${{secrets.twine_password}}
      DOCKER_CLIENT_TIMEOUT: "120"
      COMPOSE_HTTP_TIMEOUT: "120"
    outputs:
      release_upload_url: ${{steps.create_gh_release.outputs.upload_url}}
    steps:
      - uses: actions/checkout@v4
      - name: Set up Python ${{matrix.py_ver}}
        uses: actions/setup-python@v5
        with:
          python-version: ${{matrix.py_ver}}
      - name: Determine package version
        run: |
          DJ_VERSION=$(grep -oP '\d+\.\d+\.\d+' datajoint/version.py)
          RELEASE_BODY=$(python -c \
            'print(open("./CHANGELOG.md").read().split("\n\n")[1].split("\n", 1)[1])' \
          )
          echo "DJ_VERSION=${DJ_VERSION}" >> $GITHUB_ENV
          echo "RELEASE_BODY<<EOF" >> $GITHUB_ENV
          echo "$RELEASE_BODY" >> $GITHUB_ENV
          echo "EOF" >> $GITHUB_ENV
      - name: Create GH release
        id: create_gh_release
        uses: actions/create-release@v1
        env:
          GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
        with:
          tag_name: ${{env.DJ_VERSION}}
          release_name: Release ${{env.DJ_VERSION}}
          body: ${{env.RELEASE_BODY}}
          prerelease: false
          draft: false
      - name: Fetch pip artifacts
        uses: actions/download-artifact@v4.1.7
        with:
          name: pip-datajoint-${{env.DJ_VERSION}}
          path: dist
      - name: Determine pip artifact paths
        run: |
          echo "DJ_WHEEL_PATH=$(ls dist/datajoint-*.whl)" >> $GITHUB_ENV
          echo "DJ_SDIST_PATH=$(ls dist/datajoint-*.tar.gz)" >> $GITHUB_ENV
      - name: Upload pip wheel asset to release
        uses: actions/upload-release-asset@v1
        env:
          GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
        with:
          upload_url: ${{steps.create_gh_release.outputs.upload_url}}
          asset_path: ${{env.DJ_WHEEL_PATH}}
          asset_name: pip-datajoint-${{env.DJ_VERSION}}.whl
          asset_content_type: application/zip
      - name: Upload pip sdist asset to release
        uses: actions/upload-release-asset@v1
        env:
          GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
        with:
          upload_url: ${{steps.create_gh_release.outputs.upload_url}}
          asset_path: ${{env.DJ_SDIST_PATH}}
          asset_name: pip-datajoint-${{env.DJ_VERSION}}.tar.gz
          asset_content_type: application/gzip
      - name: Publish pip release
        run: |
          export HOST_UID=$(id -u)
          docker compose -f docker-compose-build.yaml run \
            -e TWINE_USERNAME=${TWINE_USERNAME} -e TWINE_PASSWORD=${TWINE_PASSWORD} app \
            sh -c "pip install twine && python -m twine upload dist/*"
      - name: Login to DockerHub
        uses: docker/login-action@v1
        with:
          username: ${{secrets.docker_username}}
          password: ${{secrets.docker_password}}
      - name: Publish image
        run: |
          IMAGE=$(docker images --filter "reference=datajoint/datajoint*" --format "{{.Repository}}")
          TAG=$(docker images --filter "reference=datajoint/datajoint*" --format "{{.Tag}}")
          docker push "${IMAGE}:${TAG}"
          docker tag "${IMAGE}:${TAG}" "${IMAGE}:${TAG}-${GITHUB_SHA:0:7}"
          docker push "${IMAGE}:${TAG}-${GITHUB_SHA:0:7}"
          [ "$PY_VER" == "3.9" ] && [ "$DISTRO" == "debian" ] \
                  && docker tag "${IMAGE}:${TAG}" "${IMAGE}:latest" \
                  && docker push "${IMAGE}:latest" \
              || echo "skipping 'latest' tag..."