diff --git a/NodeBase/Dockerfile b/NodeBase/Dockerfile index 90e45298e..c99fce864 100644 --- a/NodeBase/Dockerfile +++ b/NodeBase/Dockerfile @@ -133,6 +133,14 @@ RUN chmod -R 775 ${HOME} /tmp/.X11-unix \ && chgrp -R 0 ${HOME} /tmp/.X11-unix \ && chmod -R g=u ${HOME} /tmp/.X11-unix +#============================================ +# Shared cleanup script environment variables +#============================================ +ENV SE_ENABLE_BROWSER_LEFTOVERS_CLEANUP false +ENV SE_BROWSER_LEFTOVERS_INTERVAL_SECS 3600 +ENV SE_BROWSER_LEFTOVERS_PROCESSES_SECS 1200 +ENV SE_BROWSER_LEFTOVERS_TEMPFILES_DAYS 1 + #=================================================== # Run the following commands as non-privileged user #=================================================== diff --git a/NodeChrome/Dockerfile b/NodeChrome/Dockerfile index 5c31e6e5e..432265f91 100644 --- a/NodeChrome/Dockerfile +++ b/NodeChrome/Dockerfile @@ -53,6 +53,12 @@ RUN if [ ! -z "$CHROME_DRIVER_VERSION" ]; \ && chmod 755 /opt/selenium/chromedriver-$CHROME_DRIVER_VERSION \ && ln -fs /opt/selenium/chromedriver-$CHROME_DRIVER_VERSION /usr/bin/chromedriver +#============================================ +# Chrome cleanup script and supervisord file +#============================================ +COPY chrome-cleanup.sh /opt/bin/chrome-cleanup.sh +COPY chrome-cleanup.conf /etc/supervisor/conf.d/chrome-cleanup.conf + USER ${SEL_UID} #============================================ diff --git a/NodeChrome/chrome-cleanup.conf b/NodeChrome/chrome-cleanup.conf new file mode 100644 index 000000000..0ab6a1858 --- /dev/null +++ b/NodeChrome/chrome-cleanup.conf @@ -0,0 +1,21 @@ +; Documentation of this file format -> http://supervisord.org/configuration.html + +; Priority 0 - xvfb & fluxbox, 5 - x11vnc, 10 - noVNC, 15 - selenium-node + +[program:browserleftoverscleanup] +priority=0 +command=bash -c "if [ ${SE_ENABLE_BROWSER_LEFTOVERS_CLEANUP} = "true" ]; then /opt/bin/chrome-cleanup.sh; fi" +autostart=true +exitcodes=0 +autorestart=unexpected + +;Logs +redirect_stderr=false +stdout_logfile=/var/log/supervisor/browser-leftover-cleanup-stdout.log +stderr_logfile=/var/log/supervisor/browser-leftover-cleanup-stderr.log +stdout_logfile_maxbytes=50MB +stderr_logfile_maxbytes=50MB +stdout_logfile_backups=5 +stderr_logfile_backups=5 +stdout_capture_maxbytes=50MB +stderr_capture_maxbytes=50MB diff --git a/NodeChrome/chrome-cleanup.sh b/NodeChrome/chrome-cleanup.sh new file mode 100755 index 000000000..d90dcf40f --- /dev/null +++ b/NodeChrome/chrome-cleanup.sh @@ -0,0 +1,37 @@ +#!/bin/bash + +# Return error exit code in case of any failure, so supervisord will restart the script +set -e + +cleanup_stuck_chrome_processes() { + echo -n "Killing Chrome processes older than ${SE_BROWSER_LEFTOVERS_PROCESSES_SECS} seconds... " + ps -e -o pid,etimes,command | grep -v grep | grep chrome/chrome | awk '{if($2>'${SE_BROWSER_LEFTOVERS_PROCESSES_SECS}') print $0}' | awk '{print $1}' | xargs -r kill -9 + echo "DONE." +} + +cleanup_tmp_chrome_files() { + echo -n "Deleting all Chrome files in /tmp... " + find /tmp -name ".com.google.Chrome.*" -type d -mtime +${SE_BROWSER_LEFTOVERS_TEMPFILES_DAYS} -exec rm -rf "{}" + + echo "DONE." +} + +echo "Chrome cleanup script init with parameters: SE_BROWSER_LEFTOVERS_TEMPFILES_DAYS=${SE_BROWSER_LEFTOVERS_TEMPFILES_DAYS}, SE_BROWSER_LEFTOVERS_PROCESSES_SECS=${SE_BROWSER_LEFTOVERS_PROCESSES_SECS}, SE_BROWSER_LEFTOVERS_INTERVAL_SECS=${SE_BROWSER_LEFTOVERS_INTERVAL_SECS}." + +# Start the main loop +while : +do + echo "Starting cleanup daemon script." + + # Clean up stuck processes + cleanup_stuck_chrome_processes + + # Wait a few seconds for the processes to stop before removing files + sleep 5 + + # Clean up temporary files + cleanup_tmp_chrome_files + + # Go to sleep for 1 hour + echo "Cleanup daemon sleeping for ${SE_BROWSER_LEFTOVERS_INTERVAL_SECS} seconds." + sleep ${SE_BROWSER_LEFTOVERS_INTERVAL_SECS} +done diff --git a/NodeEdge/Dockerfile b/NodeEdge/Dockerfile index 8140e7406..4f6ddeb0f 100644 --- a/NodeEdge/Dockerfile +++ b/NodeEdge/Dockerfile @@ -46,6 +46,12 @@ RUN if [ -z "$EDGE_DRIVER_VERSION" ]; \ && chmod 755 /opt/selenium/msedgedriver-$EDGE_DRIVER_VERSION \ && ln -fs /opt/selenium/msedgedriver-$EDGE_DRIVER_VERSION /usr/bin/msedgedriver +#============================================ +# Edge cleanup script and supervisord file +#============================================ +COPY edge-cleanup.sh /opt/bin/edge-cleanup.sh +COPY edge-cleanup.conf /etc/supervisor/conf.d/edge-cleanup.conf + USER ${SEL_UID} #============================================ diff --git a/NodeEdge/edge-cleanup.conf b/NodeEdge/edge-cleanup.conf new file mode 100644 index 000000000..3e02cd0dd --- /dev/null +++ b/NodeEdge/edge-cleanup.conf @@ -0,0 +1,21 @@ +; Documentation of this file format -> http://supervisord.org/configuration.html + +; Priority 0 - xvfb & fluxbox, 5 - x11vnc, 10 - noVNC, 15 - selenium-node + +[program:browserleftoverscleanup] +priority=0 +command=bash -c "if [ ${SE_ENABLE_BROWSER_LEFTOVERS_CLEANUP} = "true" ]; then /opt/bin/edge-cleanup.sh; fi" +autostart=true +exitcodes=0 +autorestart=unexpected + +;Logs +redirect_stderr=false +stdout_logfile=/var/log/supervisor/browser-leftover-cleanup-stdout.log +stderr_logfile=/var/log/supervisor/browser-leftover-cleanup-stderr.log +stdout_logfile_maxbytes=50MB +stderr_logfile_maxbytes=50MB +stdout_logfile_backups=5 +stderr_logfile_backups=5 +stdout_capture_maxbytes=50MB +stderr_capture_maxbytes=50MB diff --git a/NodeEdge/edge-cleanup.sh b/NodeEdge/edge-cleanup.sh new file mode 100755 index 000000000..06298cb67 --- /dev/null +++ b/NodeEdge/edge-cleanup.sh @@ -0,0 +1,37 @@ +#!/bin/bash + +# Return error exit code in case of any failure, so supervisord will restart the script +set -e + +cleanup_stuck_edge_processes() { + echo -n "Killing Edge processes older than ${SE_BROWSER_LEFTOVERS_PROCESSES_SECS} seconds... " + ps -e -o pid,etimes,command | grep -v grep | grep msedge/msedge | awk '{if($2>'${SE_BROWSER_LEFTOVERS_PROCESSES_SECS}') print $0}' | awk '{print $1}' | xargs -r kill -9 + echo "DONE." +} + +cleanup_tmp_edge_files() { + echo -n "Deleting all Edge files in /tmp... " + find /tmp -name ".com.microsoft.Edge.*" -type d -mtime +${SE_BROWSER_LEFTOVERS_TEMPFILES_DAYS} -exec rm -rf "{}" + + echo "DONE." +} + +echo "Edge cleanup script init with parameters: SE_BROWSER_LEFTOVERS_TEMPFILES_DAYS=${SE_BROWSER_LEFTOVERS_TEMPFILES_DAYS}, SE_BROWSER_LEFTOVERS_PROCESSES_SECS=${SE_BROWSER_LEFTOVERS_PROCESSES_SECS}, SE_BROWSER_LEFTOVERS_INTERVAL_SECS=${SE_BROWSER_LEFTOVERS_INTERVAL_SECS}." + +# Start the main loop +while : +do + echo "Starting cleanup daemon script." + + # Clean up stuck processes + cleanup_stuck_edge_processes + + # Wait a few seconds for the processes to stop before removing files + sleep 5 + + # Clean up temporary files + cleanup_tmp_edge_files + + # Go to sleep for 1 hour + echo "Cleanup daemon sleeping for ${SE_BROWSER_LEFTOVERS_INTERVAL_SECS} seconds." + sleep ${SE_BROWSER_LEFTOVERS_INTERVAL_SECS} +done diff --git a/NodeFirefox/Dockerfile b/NodeFirefox/Dockerfile index 92d2684c5..7dc0e4340 100644 --- a/NodeFirefox/Dockerfile +++ b/NodeFirefox/Dockerfile @@ -36,6 +36,12 @@ RUN GK_VERSION=$(if [ ${GECKODRIVER_VERSION:-latest} = "latest" ]; then echo "0. && chmod 755 /opt/geckodriver-$GK_VERSION \ && ln -fs /opt/geckodriver-$GK_VERSION /usr/bin/geckodriver +#============================================ +# Firefox cleanup script and supervisord file +#============================================ +COPY firefox-cleanup.sh /opt/bin/firefox-cleanup.sh +COPY firefox-cleanup.conf /etc/supervisor/conf.d/firefox-cleanup.conf + USER ${SEL_UID} #============================================ diff --git a/NodeFirefox/firefox-cleanup.conf b/NodeFirefox/firefox-cleanup.conf new file mode 100644 index 000000000..43aaf6f20 --- /dev/null +++ b/NodeFirefox/firefox-cleanup.conf @@ -0,0 +1,21 @@ +; Documentation of this file format -> http://supervisord.org/configuration.html + +; Priority 0 - xvfb & fluxbox, 5 - x11vnc, 10 - noVNC, 15 - selenium-node + +[program:browserleftoverscleanup] +priority=0 +command=bash -c "if [ ${SE_ENABLE_BROWSER_LEFTOVERS_CLEANUP} = "true" ]; then /opt/bin/firefox-cleanup.sh; fi" +autostart=true +exitcodes=0 +autorestart=unexpected + +;Logs +redirect_stderr=false +stdout_logfile=/var/log/supervisor/browser-leftover-cleanup-stdout.log +stderr_logfile=/var/log/supervisor/browser-leftover-cleanup-stderr.log +stdout_logfile_maxbytes=50MB +stderr_logfile_maxbytes=50MB +stdout_logfile_backups=5 +stderr_logfile_backups=5 +stdout_capture_maxbytes=50MB +stderr_capture_maxbytes=50MB diff --git a/NodeFirefox/firefox-cleanup.sh b/NodeFirefox/firefox-cleanup.sh new file mode 100755 index 000000000..978b7dfe1 --- /dev/null +++ b/NodeFirefox/firefox-cleanup.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +# Return error exit code in case of any failure, so supervisord will restart the script +set -e + +cleanup_stuck_firefox_processes() { + echo -n "Killing Firefox processes older than ${SE_BROWSER_LEFTOVERS_PROCESSES_SECS} seconds... " + ps -e -o pid,etimes,command | grep -v grep | grep firefox-bin | awk '{if($2>'${SE_BROWSER_LEFTOVERS_PROCESSES_SECS}') print $0}' | awk '{print $1}' | xargs -r kill -9 + echo "DONE." +} + +echo "Firefox cleanup script init with parameters: SE_BROWSER_LEFTOVERS_PROCESSES_SECS=${SE_BROWSER_LEFTOVERS_PROCESSES_SECS}, SE_BROWSER_LEFTOVERS_INTERVAL_SECS=${SE_BROWSER_LEFTOVERS_INTERVAL_SECS}." + +# Start the main loop +while : +do + echo "Starting cleanup daemon script." + + # Clean up stuck processes + cleanup_stuck_firefox_processes + + # Go to sleep for 1 hour + echo "Cleanup daemon sleeping for ${SE_BROWSER_LEFTOVERS_INTERVAL_SECS} seconds." + sleep ${SE_BROWSER_LEFTOVERS_INTERVAL_SECS} +done diff --git a/README.md b/README.md index 311b7a67d..d7d8d7d55 100644 --- a/README.md +++ b/README.md @@ -944,6 +944,38 @@ $ docker run -e SE_DRAIN_AFTER_SESSION_COUNT=5 --shm-size="2g" selenium/standalo With the previous command, the Standalone container will shut down after 5 sessions have been executed. +### Automatic browser leftovers cleanup + +In long-running containers, it can happen that browsers leave some leftovers. These can be stuck browser processes +of jobs that have already finished but failed to fully stop the browser, or temporary files written to the `/tmp` +file system (notably on Chrome-based browsers). To avoid these filling up resources like process IDs and file system +usage in the container, there is an automatic cleanup script running every hour in the node containers. This will +clean up old processes and old temporary files. By default, this is disabled. When enabled, this will clean up browsers +running for longer than 20 minutes, and files older than 1 day. These can be enabled and tweaked with the following +environment variables: + +* `SE_ENABLE_BROWSER_LEFTOVERS_CLEANUP`: default value `false`, set to `true` to enable the cleanup. +* `SE_BROWSER_LEFTOVERS_INTERVAL_SECS`: default value `3600` (1 hour), cleanup interval in seconds. +* `SE_BROWSER_LEFTOVERS_PROCESSES_SECS`: default value `1200` (2 hours), browsers running for longer than this time will be killed. +* `SE_BROWSER_LEFTOVERS_TEMPFILES_DAYS`: default value `1` (1 day), files generated by Chrome-based browsers in `/tmp` will be removed after these number of days (ignored when using Firefox). + +If you use Selenium for long-running sessions and expect browsers to be running for longer than 20 minutes, either do +not set `SE_ENABLE_BROWSER_LEFTOVERS_CLEANUP` to `true` (leave the default value of `false`), or tweak +`SE_BROWSER_LEFTOVERS_PROCESSES_SECS` to set a value higher than your expected long-running browser processes. + +``` bash +$ docker run -e SE_ENABLE_BROWSER_LEFTOVERS_CLEANUP=true --shm-size="2g" selenium/node-chrome:4.18.1-20240224 +``` + +With the previous command, the cleanup will be enabled with the default timings. + +``` bash +$ docker run -e SE_ENABLE_BROWSER_LEFTOVERS_CLEANUP=true -e SE_BROWSER_LEFTOVERS_INTERVAL_SECS=7200 -e SE_BROWSER_LEFTOVERS_PROCESSES_SECS=3600 -e SE_BROWSER_LEFTOVERS_TEMPFILES_DAYS=2 --shm-size="2g" selenium/node-chrome:4.18.1-20240224 +``` + +With the previous command, the cleanup will be enabled, but will run every 2 hours (instead of 1), will kill browsers +running longer than 1 hour (instead of 20 minutes), and will remove temp files older than 2 days (instead of 1). + ___ ## Building the images