From eb19c217d1fb140d4b71757b3f096a76277cd97d Mon Sep 17 00:00:00 2001 From: Martin Pitt Date: Tue, 13 Feb 2024 16:46:51 +0100 Subject: [PATCH 1/4] tasks: Install dbus-daemon and valgrind These are necessary for running cockpit's unit tests. This is useful for human developers, and also paves the way for moving the unit test CI run to the tasks container. --- tasks/Containerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tasks/Containerfile b/tasks/Containerfile index 307caf41..64d309b5 100644 --- a/tasks/Containerfile +++ b/tasks/Containerfile @@ -8,6 +8,7 @@ RUN dnf -y update && \ byobu \ chromium-headless \ curl \ + dbus-daemon \ dbus-glib \ diffstat \ expect \ @@ -50,6 +51,7 @@ RUN dnf -y update && \ socat \ strace \ tar \ + valgrind \ vim-enhanced \ virt-install && \ curl -o /tmp/cockpit.spec -s https://raw.githubusercontent.com/cockpit-project/cockpit/main/tools/cockpit.spec && \ From aa49c25b7f728459b125f1c9a30a19bc2999903c Mon Sep 17 00:00:00 2001 From: Martin Pitt Date: Wed, 14 Feb 2024 10:00:07 +0100 Subject: [PATCH 2/4] tasks: Move $NODE_EXTRA_CA_CERTS to general container environment This has tripped me up time and again when `podman exec`'ing into a container, as in that shell the variable wouldn't be set. This also helps with moving the initialization to a separate script. --- tasks/cockpit-tasks | 3 --- tasks/install-service | 5 +++++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/tasks/cockpit-tasks b/tasks/cockpit-tasks index 9d908a2d..dae2fbf9 100755 --- a/tasks/cockpit-tasks +++ b/tasks/cockpit-tasks @@ -17,9 +17,6 @@ if [ -n "${NPM_REGISTRY:-}" ]; then npm config set registry "$NPM_REGISTRY" echo "Set NPM registry to $NPM_REGISTRY" fi -if [ -r /secrets/npm-registry.crt ]; then - export NODE_EXTRA_CA_CERTS=/secrets/npm-registry.crt -fi # prone to timeouts and errors with lots of parallel containers npm config set fetch-retries 6 diff --git a/tasks/install-service b/tasks/install-service index c278750d..f7602a0d 100755 --- a/tasks/install-service +++ b/tasks/install-service @@ -22,6 +22,10 @@ mkdir -p $SECRETS/tasks $SECRETS/webhook $CACHE chown -R 1111:1111 $SECRETS $CACHE chcon -R -t container_file_t $SECRETS $CACHE +if [ -e "${SECRETS}/tasks/npm-registry.crt" ]; then + NODE_EXTRA_CA_CERTS=/secrets/npm-registry.crt +fi + if [ $INSTANCES -eq 1 ]; then # just use the hostname without prefix CONTAINER_HOSTNAME="%l" @@ -62,6 +66,7 @@ ExecStart=/usr/bin/podman run --name=cockpit-tasks-%i --hostname=${CONTAINER_HOS --volume=\${TEST_SECRETS}/webhook:/run/secrets/webhook:ro \ --volume=${IMAGE_STORES}:/work/.config/cockpit-dev/image-stores:ro \ --env=NPM_REGISTRY=\${NPM_REGISTRY} \ + --env=NODE_EXTRA_CA_CERTS=${NODE_EXTRA_CA_CERTS:-} \ --env=TEST_JOBS=\${TEST_JOBS} \ --env=TEST_NOTIFICATION_MX=\${TEST_NOTIFICATION_MX} \ --env=TEST_NOTIFICATION_TO=\${TEST_NOTIFICATION_TO} \ From 2947dfd19a87775ddc5b6bad7d6bd5a692c7f03b Mon Sep 17 00:00:00 2001 From: Martin Pitt Date: Tue, 13 Feb 2024 16:52:30 +0100 Subject: [PATCH 3/4] tasks: Move config and secret setup to separate script This paves the way for using the tasks containter in "single job" mode without the `cockpit-tasks` mainloop. --- tasks/Containerfile | 2 +- tasks/cockpit-tasks | 36 ++---------------------------------- tasks/setup-tasks | 37 +++++++++++++++++++++++++++++++++++++ 3 files changed, 40 insertions(+), 35 deletions(-) create mode 100755 tasks/setup-tasks diff --git a/tasks/Containerfile b/tasks/Containerfile index 64d309b5..dae025c0 100644 --- a/tasks/Containerfile +++ b/tasks/Containerfile @@ -60,7 +60,7 @@ RUN dnf -y update && \ dnf clean all && \ pip install ruff -COPY cockpit-tasks install-service webhook github_handler.py /usr/local/bin/ +COPY setup-tasks cockpit-tasks install-service webhook github_handler.py /usr/local/bin/ RUN groupadd -g 1111 -r user && useradd -r -g user -u 1111 user --home-dir /work && \ mkdir -p /usr/local/bin /secrets /cache/images /cache/github && \ diff --git a/tasks/cockpit-tasks b/tasks/cockpit-tasks index dae2fbf9..388fc274 100755 --- a/tasks/cockpit-tasks +++ b/tasks/cockpit-tasks @@ -2,47 +2,15 @@ set -eux +setup-tasks + COCKPIT_BOTS_REPO=${COCKPIT_BOTS_REPO:-https://github.com/cockpit-project/bots} COCKPIT_BOTS_BRANCH=${COCKPIT_BOTS_BRANCH:-main} -# ensure we have a passwd entry for random UIDs -# https://docs.openshift.com/container-platform/3.7/creating_images/guidelines.html -if ! whoami && [ -w /etc/passwd ]; then - echo "user:x:$(id -u):0:random uid:/work:/sbin/nologin" >> /etc/passwd - export HOME=/work -fi - -# set up custom NPM registry -if [ -n "${NPM_REGISTRY:-}" ]; then - npm config set registry "$NPM_REGISTRY" - echo "Set NPM registry to $NPM_REGISTRY" -fi - -# prone to timeouts and errors with lots of parallel containers -npm config set fetch-retries 6 -npm config set fetch-timeout 600000 -npm config set fetch-retry-mintimeout 60000 -npm config set maxsockets 3 - -# set up S3 keys for OpenShift secrets volume -if [ ! -d /secrets/s3-keys ]; then - # then our container symlink will point into the void, replace it with a directory and set up all files that we can find - rm ~/.config/cockpit-dev/s3-keys - mkdir ~/.config/cockpit-dev/s3-keys - for f in /secrets/s3-keys--*; do - [ -e "$f" ] || continue # non-matching glob - ln -s "$f" ~/.config/cockpit-dev/s3-keys/"${f#*--}" - done -fi - # let's just do our work in the current directory WORKDIR="$PWD" BOTS_DIR="$WORKDIR"/bots -# Set up github user and token -git config --global credential.helper store -echo "https://cockpituous:$(cat ~/.config/github-token)@github.com" > ~/.git-credentials - echo "Starting testing" function update_bots() { diff --git a/tasks/setup-tasks b/tasks/setup-tasks new file mode 100755 index 00000000..e1bf632f --- /dev/null +++ b/tasks/setup-tasks @@ -0,0 +1,37 @@ +#!/bin/sh +# set up configuration and secrets for running tasks +set -eux + +# ensure we have a passwd entry for random UIDs +# https://docs.openshift.com/container-platform/3.7/creating_images/guidelines.html +if ! whoami && [ -w /etc/passwd ]; then + echo "user:x:$(id -u):0:random uid:/work:/sbin/nologin" >> /etc/passwd + export HOME=/work +fi + +# set up custom NPM registry +if [ -n "${NPM_REGISTRY:-}" ]; then + npm config set registry "$NPM_REGISTRY" + echo "Set NPM registry to $NPM_REGISTRY" +fi + +# prone to timeouts and errors with lots of parallel containers +npm config set fetch-retries 6 +npm config set fetch-timeout 600000 +npm config set fetch-retry-mintimeout 60000 +npm config set maxsockets 3 + +# set up S3 keys for OpenShift secrets volume +if [ ! -d /secrets/s3-keys ]; then + # then our container symlink will point into the void, replace it with a directory and set up all files that we can find + rm ~/.config/cockpit-dev/s3-keys + mkdir ~/.config/cockpit-dev/s3-keys + for f in /secrets/s3-keys--*; do + [ -e "$f" ] || continue # non-matching glob + ln -s "$f" ~/.config/cockpit-dev/s3-keys/"${f#*--}" + done +fi + +# Set up github user and token +git config --global credential.helper store +echo "https://cockpituous:$(cat ~/.config/github-token)@github.com" > ~/.git-credentials From 633f2187f22c2f162bc467db9adbffd3551d0f7c Mon Sep 17 00:00:00 2001 From: Martin Pitt Date: Tue, 13 Feb 2024 17:59:14 +0100 Subject: [PATCH 4/4] tasks: Create a id 1001 user We want to run cockpit's unit tests in the tasks container. They require the current user to have an /etc/passwd entry, and the tests to not run as root. GitHub workflows run as UID 1001, so add a user for that. --- tasks/Containerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/tasks/Containerfile b/tasks/Containerfile index dae025c0..590b349e 100644 --- a/tasks/Containerfile +++ b/tasks/Containerfile @@ -63,6 +63,7 @@ RUN dnf -y update && \ COPY setup-tasks cockpit-tasks install-service webhook github_handler.py /usr/local/bin/ RUN groupadd -g 1111 -r user && useradd -r -g user -u 1111 user --home-dir /work && \ + groupadd -g 1001 -r github && useradd -r --no-create-home -g github -u 1001 github && \ mkdir -p /usr/local/bin /secrets /cache/images /cache/github && \ mkdir -p /work/.config /work/.config/cockpit-dev /work/.ssh /work/.cache /work/.rhel && \ printf '[user]\n\t\nemail = cockpituous@cockpit-project.org\n\tname = Cockpituous\n[cockpit "bots"]\n\timages-data-dir = /cache/images\n' >/work/.gitconfig && \