Don't stop after only 2 test failures #167
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # This is a GitHub workflow defining a set of jobs with a set of steps. | |
| # ref: https://docs.github.com/en/actions/learn-github-actions/workflow-syntax-for-github-actions | |
| # | |
| name: Tests | |
| on: | |
| pull_request: | |
| paths-ignore: | |
| - "**.md" | |
| - "**.rst" | |
| - "docs/**" | |
| - "examples/**" | |
| - ".github/workflows/**" | |
| - "!.github/workflows/test.yml" | |
| push: | |
| paths-ignore: | |
| - "**.md" | |
| - "**.rst" | |
| - "docs/**" | |
| - "examples/**" | |
| - ".github/workflows/**" | |
| - "!.github/workflows/test.yml" | |
| branches-ignore: | |
| - "dependabot/**" | |
| - "pre-commit-ci-update-config" | |
| - "update-*" | |
| workflow_dispatch: | |
| jobs: | |
| lint: | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: install requirements | |
| run: pip install ruamel.yaml | |
| - name: check embedded chart code | |
| run: ./ci/check_embedded_chart_code.py | |
| # Most of the "main", "auth" and "helm" jobs are the same and only differ | |
| # in small things. Unfortunately there is no easy way to share steps between | |
| # jobs or have "template" jobs, so we use `if` conditions on steps | |
| tests: | |
| runs-on: ubuntu-22.04 | |
| timeout-minutes: 10 | |
| permissions: | |
| contents: read | |
| env: | |
| GITHUB_ACCESS_TOKEN: "${{ secrets.github_token }}" | |
| strategy: | |
| # keep running so we can see if tests with other k3s/k8s/helm versions pass | |
| fail-fast: false | |
| matrix: | |
| k3s-channel: | |
| # Available channels: https://github.com/k3s-io/k3s/blob/HEAD/channel.yaml | |
| # Don't use "latest", instead bump it using a PR so we know when a new version breaks BinderHub | |
| - v1.29 | |
| test: | |
| - main | |
| - auth | |
| test-variation: | |
| - "" | |
| include: | |
| # Chart.yaml contains the chart's oldest supported k8s version, we | |
| # test against that and the oldest known supported helm cli version | |
| # which isn't documented anywhere, but is currently at 3.5.0. We also | |
| # test with the latest k8s and helm versions. | |
| # | |
| - k3s-channel: v1.23 | |
| helm-version: v3.5.0 | |
| test: helm | |
| test-variation: dind | |
| local-chart-extra-args: >- | |
| --values testing/k8s-binder-k8s-hub/binderhub-chart+dind.yaml | |
| --set config.BinderHub.image_prefix=$REGISTRY_HOST/test/ | |
| --set registry.url=http://$REGISTRY_HOST | |
| - k3s-channel: v1.29 | |
| test: helm | |
| test-variation: pink | |
| local-chart-extra-args: >- | |
| --values testing/k8s-binder-k8s-hub/binderhub-chart+pink.yaml | |
| --set config.BinderHub.image_prefix=$REGISTRY_HOST/test/ | |
| --set registry.url=http://$REGISTRY_HOST | |
| - k3s-channel: v1.29 | |
| test: helm | |
| test-variation: upgrade | |
| # upgrade-from represents a release channel, see: https://jupyterhub.github.io/helm-chart/info.json | |
| upgrade-from: dev | |
| upgrade-from-extra-args: "" | |
| services: | |
| registry: | |
| image: docker.io/library/registry:latest | |
| ports: | |
| - 5000:5000 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| # chartpress requires the full history | |
| fetch-depth: 0 | |
| - name: Set registry host | |
| if: matrix.test-variation == 'dind' || matrix.test-variation == 'pink' | |
| run: | | |
| REGISTRY_HOST=$(hostname -I | awk '{print $1}'):5000 | |
| echo REGISTRY_HOST="$REGISTRY_HOST" >> $GITHUB_ENV | |
| # Allow k3s to pull from private registry | |
| # https://docs.k3s.io/installation/private-registry | |
| sudo mkdir -p /etc/rancher/k3s/ | |
| cat << EOF | sudo tee /etc/rancher/k3s/registries.yaml | |
| mirrors: | |
| "$REGISTRY_HOST": | |
| endpoint: | |
| - "http://$REGISTRY_HOST" | |
| EOF | |
| - uses: jupyterhub/action-k3s-helm@v4 | |
| with: | |
| k3s-channel: ${{ matrix.k3s-channel }} | |
| helm-version: ${{ matrix.helm-version }} | |
| metrics-enabled: false | |
| traefik-enabled: false | |
| docker-enabled: ${{ matrix.test-variation != 'dind' && matrix.test-variation != 'pink' }} | |
| - name: Setup OS level dependencies | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install --yes \ | |
| build-essential \ | |
| curl \ | |
| libcurl4-openssl-dev \ | |
| libssl-dev | |
| - uses: actions/setup-node@v4 | |
| id: setup-node | |
| with: | |
| node-version: "18" | |
| - name: Cache npm | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/.npm | |
| key: node-${{ steps.setup-node.outputs.node-version }}-${{ hashFiles('**/package.json') }}-${{ github.job }} | |
| - name: Run webpack to build static assets | |
| run: | | |
| npm install | |
| npm run webpack | |
| - uses: actions/setup-python@v5 | |
| id: setup-python | |
| with: | |
| python-version: "3.11" | |
| - name: Cache pip | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/.cache/pip | |
| key: python-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/*requirements.txt') }}-${{ github.job }} | |
| - name: Update pip | |
| run: | | |
| pip install --upgrade pip | |
| pip install --upgrade setuptools wheel | |
| - name: Setup Python package dependencies | |
| run: | | |
| pip install --no-binary pycurl -r dev-requirements.txt -r helm-chart/images/binderhub/requirements.txt | |
| pip install -e . | |
| - name: Install JupyterHub chart for main tests | |
| if: matrix.test == 'main' | |
| run: | | |
| ./testing/local-binder-k8s-hub/install-jupyterhub-chart | |
| - name: Install JupyterHub chart for auth tests | |
| if: matrix.test == 'auth' | |
| run: | | |
| ./testing/local-binder-k8s-hub/install-jupyterhub-chart --auth | |
| - name: Build binderhub wheel | |
| if: matrix.test == 'helm' | |
| run: | | |
| python3 -m build . | |
| - name: Use chartpress to create the helm chart | |
| if: matrix.test == 'helm' | |
| run: | | |
| export DOCKER_BUILDKIT=1 | |
| CHARTPRESS_ARGS= | |
| if [ "${{ matrix.test-variation }}" = "dind" -o "${{ matrix.test-variation }}" = "pink" ]; then | |
| CHARTPRESS_ARGS="--image-prefix localhost:5000/binderhub- --push" | |
| # Allow the pods to push to the non-https GitHub workflow registry | |
| envsubst < testing/k8s-binder-k8s-hub/cm-insecure-registries-${{ matrix.test-variation }}.yaml | kubectl apply -f - | |
| fi | |
| # Use chartpress to create the helm chart and build its images | |
| helm dependency update ./helm-chart/binderhub | |
| (cd helm-chart && chartpress $CHARTPRESS_ARGS) | |
| git --no-pager diff --color=always | |
| - name: Generate values.schema.json from schema.yaml | |
| if: matrix.test == 'helm' | |
| run: | | |
| tools/generate-json-schema.py | |
| - name: "Helm template --validate (with lint-and-validate-values.yaml)" | |
| if: matrix.test == 'helm' | |
| run: | | |
| helm template --validate binderhub-test helm-chart/binderhub \ | |
| --values tools/templates/lint-and-validate-values.yaml | |
| - name: "Helm template --validate for dind (with lint-and-validate-values.yaml)" | |
| if: matrix.test == 'helm' | |
| run: | | |
| helm template --validate binderhub-test helm-chart/binderhub \ | |
| --values tools/templates/lint-and-validate-values.yaml \ | |
| --set imageBuilderType=dind | |
| - name: "Helm template --validate for pink (with lint-and-validate-values.yaml)" | |
| if: matrix.test == 'helm' | |
| run: | | |
| helm template --validate binderhub-test helm-chart/binderhub \ | |
| --values tools/templates/lint-and-validate-values.yaml \ | |
| --set imageBuilderType=pink | |
| - name: Validate the chart against the k8s API | |
| if: matrix.test == 'helm' | |
| run: | | |
| helm template --validate binderhub-test helm-chart/binderhub \ | |
| --values testing/k8s-binder-k8s-hub/binderhub-chart-config.yaml \ | |
| --set config.BinderHub.hub_url=http://localhost:30902 \ | |
| --set config.GitHubRepoProvider.access_token=$GITHUB_ACCESS_TOKEN | |
| - name: "(Upgrade) Install ${{ matrix.upgrade-from }} chart" | |
| if: matrix.test-variation == 'upgrade' | |
| run: | | |
| # NOTE: We change the directory so binderhub the chart name won't be | |
| # misunderstood as the local folder name. | |
| # | |
| # https://github.com/helm/helm/issues/9244 | |
| cd ci | |
| if [ ${{ matrix.upgrade-from }} = stable ]; then | |
| UPGRADE_FROM_VERSION=$(helm show chart --repo=https://hub.jupyter.org/helm-chart/ binderhub | yq e '.version' -) | |
| elif [ ${{ matrix.upgrade-from }} = dev ]; then | |
| UPGRADE_FROM_VERSION=$(helm show chart --devel --repo=https://hub.jupyter.org/helm-chart/ binderhub | yq e '.version' -) | |
| else | |
| UPGRADE_FROM_VERSION=${{ matrix.upgrade-from }} | |
| fi | |
| echo "UPGRADE_FROM_VERSION=$UPGRADE_FROM_VERSION" >> $GITHUB_ENV | |
| echo "" | |
| echo "Installing already released binderhub version $UPGRADE_FROM_VERSION" | |
| old_config="../testing/k8s-binder-k8s-hub/binderhub-chart-config-old.yaml" | |
| if [ -f "$old_config" ]; then | |
| echo "using old config" | |
| else | |
| old_config="../testing/k8s-binder-k8s-hub/binderhub-chart-config.yaml" | |
| fi | |
| helm install binderhub-test \ | |
| --repo https://jupyterhub.github.io/helm-chart/ binderhub \ | |
| --disable-openapi-validation \ | |
| --version=$UPGRADE_FROM_VERSION \ | |
| --values "$old_config" \ | |
| --set config.BinderHub.hub_url=http://localhost:30902 \ | |
| --set config.BinderHub.hub_url_local=http://proxy-public \ | |
| --set config.GitHubRepoProvider.access_token=$GITHUB_ACCESS_TOKEN \ | |
| ${{ matrix.upgrade-from-extra-args }} | |
| - name: "(Upgrade) Install helm diff" | |
| if: matrix.test-variation == 'upgrade' | |
| run: | | |
| helm plugin install https://github.com/databus23/helm-diff | |
| - name: "(Upgrade) Helm diff ${{ matrix.upgrade-from }} chart with local chart" | |
| if: matrix.test-variation == 'upgrade' | |
| run: | | |
| helm diff upgrade binderhub-test ./helm-chart/binderhub \ | |
| --values testing/k8s-binder-k8s-hub/binderhub-chart-config.yaml \ | |
| --set config.BinderHub.hub_url=http://localhost:30902 \ | |
| --set config.BinderHub.hub_url_local=http://proxy-public \ | |
| --set config.GitHubRepoProvider.access_token=$GITHUB_ACCESS_TOKEN \ | |
| ${{ matrix.local-chart-extra-args }} | |
| - name: "(Upgrade) Await ${{ matrix.upgrade-from }} chart" | |
| if: matrix.test-variation == 'upgrade' | |
| uses: jupyterhub/action-k8s-await-workloads@v3 | |
| with: | |
| timeout: 150 | |
| max-restarts: 1 | |
| - name: Test the workaround for an incorrect docker.sock directory | |
| if: matrix.test-variation == 'dind' | |
| run: | | |
| # {{ .Values.dind.hostSocketDir }}/{{ .Values.dind.hostSocketName }} | |
| sudo mkdir -p /var/run/dind/docker.sock | |
| - name: Install the chart | |
| if: matrix.test == 'helm' | |
| run: | | |
| helm upgrade --install binderhub-test helm-chart/binderhub \ | |
| --values testing/k8s-binder-k8s-hub/binderhub-chart-config.yaml \ | |
| --set config.BinderHub.hub_url=http://localhost:30902 \ | |
| --set config.BinderHub.hub_url_local=http://proxy-public \ | |
| --set config.GitHubRepoProvider.access_token=$GITHUB_ACCESS_TOKEN \ | |
| ${{ matrix.local-chart-extra-args }} | |
| - name: Await and curl JupyterHub | |
| run: | | |
| . ci/common | |
| await_jupyterhub | |
| echo curl http://localhost:30902/hub/api/ should print the JupyterHub version | |
| curl http://localhost:30902/hub/api/ --max-time 5 --retry 5 --retry-delay 1 --retry-connrefused --fail-with-body --retry-all-errors | |
| - name: Await and curl BinderHub | |
| if: matrix.test == 'helm' | |
| run: | | |
| . ci/common | |
| await_binderhub binderhub-test | |
| echo curl http://localhost:30901/health to check BinderHub\'s health | |
| curl http://localhost:30901/health --max-time 5 --retry 5 --retry-delay 1 --retry-connrefused --fail-with-body --retry-all-errors | |
| - name: Run main tests | |
| if: matrix.test == 'main' | |
| # running the "main" tests means "all tests that aren't auth" | |
| run: pytest -m "not auth" --cov=binderhub | |
| - name: Run auth tests | |
| if: matrix.test == 'auth' | |
| # running the "auth" tests means "all tests that are marked as auth" | |
| run: pytest -m "auth" --cov=binderhub | |
| - name: Run helm tests | |
| if: matrix.test == 'helm' | |
| run: | | |
| export BINDER_URL=http://localhost:30901 | |
| pytest --helm -m "remote" --cov=binderhub | |
| - name: Get BinderHub health and metrics outputs | |
| if: always() | |
| run: | | |
| if [ "${{ matrix.test }}" = "helm" ]; then | |
| for endpoint in versions health metrics; do | |
| echo -e "\n${endpoint}" | |
| curl http://localhost:30901/$endpoint --fail-with-body | |
| done | |
| fi | |
| # GitHub Action reference: https://github.com/jupyterhub/action-k8s-namespace-report | |
| - name: Kubernetes namespace report | |
| uses: jupyterhub/action-k8s-namespace-report@v1 | |
| if: always() | |
| with: | |
| important-workloads: > | |
| deploy/binder | |
| deploy/hub | |
| deploy/proxy | |
| daemonset/binderhub-test-dind | |
| daemonset/binderhub-test-pink | |
| # GitHub action reference: https://github.com/codecov/codecov-action | |
| - uses: codecov/codecov-action@v4 | |
| test-local: | |
| runs-on: ubuntu-22.04 | |
| timeout-minutes: 5 | |
| permissions: | |
| contents: read | |
| env: | |
| GITHUB_ACCESS_TOKEN: "${{ secrets.github_token }}" | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Setup OS level dependencies | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install --yes \ | |
| build-essential \ | |
| curl \ | |
| libcurl4-openssl-dev \ | |
| libssl-dev | |
| - uses: actions/setup-node@v4 | |
| id: setup-node | |
| with: | |
| node-version: "18" | |
| - name: Cache npm | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/.npm | |
| key: node-${{ steps.setup-node.outputs.node-version }}-${{ hashFiles('**/package.json') }}-${{ github.job }} | |
| - uses: actions/setup-python@v5 | |
| id: setup-python | |
| with: | |
| python-version: "3.11" | |
| - name: Cache pip | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/.cache/pip | |
| key: python-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/*requirements.txt') }}-${{ github.job }} | |
| - name: Update pip | |
| run: | | |
| pip install --upgrade pip | |
| pip install --upgrade setuptools wheel | |
| - name: Setup Python package dependencies | |
| # FIXME: We do "pip install ." before "pip install -e ." to ensure | |
| # binderhub/static/dist/bundle.js is built, and use "-e" to | |
| # ensure code coverage is captured. | |
| # | |
| # https://discourse.jupyter.org/t/suddenly-getting-oath-cert-error/24217/4 | |
| # pycurl wheels fail to pick up the system CA certs, so install from source | |
| # until https://github.com/pycurl/pycurl/issues/834 resolves | |
| run: | | |
| pip install -r dev-requirements.txt -r testing/local-binder-local-hub/requirements.txt | |
| pip install ".[pycurl]" --no-binary pycurl | |
| pip install -e ".[pycurl]" --no-binary pycurl | |
| - name: Setup JupyterHub NPM dependencies | |
| run: npm install -g configurable-http-proxy | |
| - name: Await and curl JupyterHub | |
| run: | | |
| cd testing/local-binder-local-hub | |
| jupyterhub --config=jupyterhub_config.py > jupyterhub.log 2>&1 & | |
| sleep 5 | |
| echo curl http://localhost:8000/hub/api/ should print the JupyterHub version | |
| curl http://localhost:8000/hub/api/ --max-time 5 --retry 5 --retry-delay 1 --retry-connrefused --fail-with-body --retry-all-errors | |
| - name: Run remote tests | |
| run: | | |
| export BINDER_URL=http://localhost:8000/services/binder/ | |
| pytest -m remote --cov=binderhub | |
| - name: Show hub logs | |
| if: always() | |
| run: cat testing/local-binder-local-hub/jupyterhub.log | |
| # GitHub action reference: https://github.com/codecov/codecov-action | |
| - uses: codecov/codecov-action@v4 |