end2end tests: update docker container version and cleanup

.ci/end2end/.dockerignore

@@ -0,0 +1 @@
+build_and_run.sh

.ci/end2end/Dockerfile

@@ -0,0 +1,19 @@
+FROM ubuntu:latest
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+COPY scripts /scripts
+
+RUN /scripts/setup_firefox.sh \
+ && /scripts/setup_chrome.sh \
+ && /scripts/setup_node.sh \
+ && apt install --yes tox git \
+# using python docs as a source of some html test data
+# need to prevent dpkg from excluding doc files...
+ && sed -i '/usr.share.doc/d' /etc/dpkg/dpkg.cfg.d/excludes && apt install --yes python3-doc \
+ && apt clean \
+ && mkdir /promnesia
+
+WORKDIR /promnesia
+
+ENTRYPOINT ["/scripts/build_and_run_tests.sh"]

.ci/end2end/build_and_run.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+set -eux
+
+# TODO assert we're running under github ci?
+# since this setup is kinda elaborate and can be somewhat unsafe to run blindly
+
+# supposed to be called from promnesia repository root
+[ -e src/promnesia ]
+[ -e extension/src ]
+
+PROMNESIA_SRC="$(pwd)"
+
+cd .ci/end2end
+
+IMAGE='promnesia_end2end_tests'
+
+docker build -t "$IMAGE" .
+
+# NOTE: dev/shm mount to prevent crashes during headless chrome
+docker run -v /dev/shm:/dev/shm --mount "type=bind,src=$PROMNESIA_SRC,dst=/promnesia_source,readonly=true" -e CI "$IMAGE" "$@"

.ci/end2end/scripts/build_and_run_tests.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+set -eux
+
+# Seems wrong to keep the whole repository in docker build context.
+# So instead, we mount the repository inside the container (into /promnesia_source)
+# (as read only to avoid messing up host files and crapping with caches etc.)
+# However to actually run tests we do need a writable directory..
+# So we copy the repo to the actual working dir here
+
+# ugh, kinda annoying -- not sure how to update source files when we change them on the host system...
+cp -R -T /promnesia_source /promnesia
+extension/.ci/build
+
+git init # todo ??? otherwise setuptools-scm fails to detect the version...
+python3 -m tox -e end2end -- "$@"

.ci/end2end/scripts/setup_chrome.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+set -eux -o pipefail
+
+apt update --yes
+
+apt install --yes wget
+
+install -d -m 0755 /etc/apt/keyrings
+wget -q https://dl.google.com/linux/linux_signing_key.pub -O- | tee /etc/apt/keyrings/linux_signing_key.pub.asc > /dev/null
+echo "deb [signed-by=/etc/apt/keyrings/linux_signing_key.pub.asc] https://dl.google.com/linux/chrome/deb/ stable main" | tee -a /etc/apt/sources.list.d/google-chrome.list > /dev/null
+
+apt update
+
+apt install --yes google-chrome-stable
+
+# sadly latest version of chrome/chromedriver isn't working due to some bugs with iframes (see install_custom_chrome)
+
+# remove the actual chrome to get it out of the way (we do want dependencies though)
+apt remove --yes google-chrome-stable
+! which google-chrome # check there is no binary (in case of virtual packages or whatever)
+
+function install_custom_chrome() {
+ ## this installs last revision that was actually working (1110897) or 113.0.5623.0
+ ## see https://bugs.chromium.org/p/chromedriver/issues/detail?id=4440
+ apt install --yes unzip
+
+ mkdir /tmp/chrome
+
+ wget -q 'https://www.googleapis.com/download/storage/v1/b/chromium-browser-snapshots/o/Linux_x64%2F1110897%2Fchrome-linux.zip?generation=1677589092014487&alt=media' \
+ -O /tmp/chrome/chrome-linux.zip
+ unzip /tmp/chrome/chrome-linux.zip -d /tmp/chrome
+ ln -sf /tmp/chrome/chrome-linux/chrome /usr/bin/google-chrome
+
+ wget -q 'https://www.googleapis.com/download/storage/v1/b/chromium-browser-snapshots/o/Linux_x64%2F1110897%2Fchromedriver_linux64.zip?generation=1677589097630198&alt=media' \
+ -O /tmp/chrome/chromedriver_linux64.zip
+ unzip /tmp/chrome/chromedriver_linux64.zip -d /tmp/chrome
+ ln -sf /tmp/chrome/chromedriver_linux64/chromedriver /usr/bin/chromedriver
+}
+
+install_custom_chrome

.ci/end2end/scripts/setup_firefox.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+set -eux -o pipefail
+
+apt update --yes
+
+apt install --yes wget
+
+# NOTE: these days ubuntu provisions firefox via snap, and it's a nightmare to make it work with webdriver
+# so we force it to use a regular package (following these instructions https://askubuntu.com/a/1510872/427470)
+install -d -m 0755 /etc/apt/keyrings
+wget -q https://packages.mozilla.org/apt/repo-signing-key.gpg -O- | tee /etc/apt/keyrings/packages.mozilla.org.asc > /dev/null
+echo "deb [signed-by=/etc/apt/keyrings/packages.mozilla.org.asc] https://packages.mozilla.org/apt mozilla main" | tee -a /etc/apt/sources.list.d/mozilla.list > /dev/null
+
+# prevent snap version from overriding:
+echo '
+Package: *
+Pin: origin packages.mozilla.org
+Pin-Priority: 1000
+' | tee /etc/apt/preferences.d/mozilla
+# to check: -- should not show anything mentioning snap
+# apt install --verbose-versions --dry-run firefox
+
+apt update
+
+apt install --yes firefox
+# NOTE: selenium should download the corresponding geckodriver itself via selenium_manager

.ci/end2end/scripts/setup_node.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+set -eux -o pipefail
+
+apt update --yes
+apt install --yes curl
+curl -fsSL https://deb.nodesource.com/setup_20.x | bash -
+apt install --yes nodejs

.github/workflows/main.yml

@@ -80,10 +80,7 @@ jobs:
  submodules: recursive
  - uses: mxschmitt/action-tmate@v3
  if: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled }}
- # NOTE: dev/shm mount to prevent crashes during headless chrome
- - run: |
- docker build -f .ci/end2end_tests.Dockerfile . -t promnesia_end2end_tests
- docker run -v /dev/shm:/dev/shm -e CI promnesia_end2end_tests -- -k chrome
+ - run: .ci/end2end/build_and_run.sh -k chrome
 
  end2end_tests_firefox:
  runs-on: ubuntu-latest
@@ -93,10 +90,7 @@ jobs:
  submodules: recursive
  - uses: mxschmitt/action-tmate@v3
  if: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled }}
- # NOTE: dev/shm mount to prevent crashes during headless chrome
- - run: |
- docker build -f .ci/end2end_tests.Dockerfile . -t promnesia_end2end_tests
- docker run -v /dev/shm:/dev/shm -e CI promnesia_end2end_tests -- -k firefox
+ - run: .ci/end2end/build_and_run.sh -k firefox
 
  install_and_run_test:
  # todo use setup-python thing?