From 838f721b8819c5357be11c9754dc3337de54da63 Mon Sep 17 00:00:00 2001 From: Viet Nguyen Duc Date: Mon, 19 Aug 2024 22:54:31 +0000 Subject: [PATCH] feat: video record and upload for standalone and dynamic grid (#2362) * test: add test for video uploader with RCLONE * Update Video/video_gridUrl.sh --------- Signed-off-by: Viet Nguyen Duc --- .circleci/config.yml | 6 + .github/workflows/docker-test.yml | 4 + Makefile | 110 +++++++--------- NodeFirefox/Dockerfile | 2 +- README.md | 18 ++- Video/Dockerfile | 2 +- Video/supervisord.conf | 6 +- Video/upload.sh | 88 +++++++------ Video/video.sh | 37 ++++-- Video/video_graphQLQuery.sh | 10 +- Video/video_gridUrl.sh | 23 ++++ Video/video_ready.py | 5 +- ...r-compose-v3-video-upload-dynamic-grid.yml | 50 ++++++++ docker-compose-v3-video-upload-standalone.yml | 120 ++++++++++++++++++ docker-compose-v3-video-upload.yml | 74 ++++++----- tests/bootstrap.sh | 2 + tests/charts/ci/base-recorder-values.yaml | 35 ++--- tests/charts/ci/local-pvc.yaml | 76 +++++++++++ tests/charts/ci/uploader.conf | 7 + tests/charts/make/chart_test.sh | 11 +- tests/docker-compose-v3-test-node-docker.yaml | 22 ++++ ...ker-compose-v3-test-standalone-docker.yaml | 24 +++- tests/docker-compose-v3-test-standalone.yml | 75 +++++++++++ tests/docker-compose-v3-test-video.yml | 27 +++- tests/standalone_docker_config.toml | 4 +- 25 files changed, 644 insertions(+), 194 deletions(-) create mode 100755 Video/video_gridUrl.sh create mode 100644 docker-compose-v3-video-upload-dynamic-grid.yml create mode 100644 docker-compose-v3-video-upload-standalone.yml create mode 100644 tests/charts/ci/uploader.conf create mode 100644 tests/docker-compose-v3-test-standalone.yml diff --git a/.circleci/config.yml b/.circleci/config.yml index 5cec8175d..41bd69c03 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -81,6 +81,12 @@ workflows: use-random-user: false platforms: linux/arm64 machine-type: ubuntu2204arm64 + - docker-test: + name: "Docker test - Video recording standalone" + test-strategy: test_video_standalone + use-random-user: false + platforms: linux/arm64 + machine-type: ubuntu2204arm64 - docker-test: name: "Docker test - Dynamic Grid" test-strategy: test_node_docker diff --git a/.github/workflows/docker-test.yml b/.github/workflows/docker-test.yml index 55f01c802..cfc174425 100644 --- a/.github/workflows/docker-test.yml +++ b/.github/workflows/docker-test.yml @@ -50,6 +50,10 @@ jobs: use-random-user: false test-video: true build-all: false + - test-strategy: test_video_standalone + use-random-user: false + test-video: true + build-all: false - test-strategy: test_node_docker use-random-user: false test-video: true diff --git a/Makefile b/Makefile index f47d474f8..d1ac10ca9 100644 --- a/Makefile +++ b/Makefile @@ -67,12 +67,14 @@ build: all ci: build test -gen_certs: +prepare_resources: rm -rf ./Base/configs/node && mkdir -p ./Base/configs/node && cp -r ./charts/selenium-grid/configs/node ./Base/configs + +gen_certs: rm -rf ./Base/certs && cp -r ./charts/selenium-grid/certs ./Base ./Base/certs/gen-cert-helper.sh -d ./Base/certs -base: gen_certs +base: prepare_resources gen_certs cd ./Base && docker buildx build --platform $(PLATFORMS) $(BUILD_ARGS) --build-arg VERSION=$(BASE_VERSION) --build-arg RELEASE=$(BASE_RELEASE) --build-arg AUTHORS=$(AUTHORS) -t $(NAME)/base:$(TAG_VERSION) . base_nightly: @@ -580,24 +582,31 @@ test_parallel: hub chrome firefox edge chromium echo SELENIUM_GRID_PROTOCOL=https >> .env ; \ echo CHART_CERT_PATH=$$(readlink -f ./videos/certs/tls.crt) >> .env ; \ export $$(cat .env | xargs) ; \ - DOCKER_DEFAULT_PLATFORM=$(PLATFORMS) docker compose --profile $(PLATFORMS) -f docker-compose-v3-test-parallel.yml up -d --no-log-prefix ; \ + DOCKER_DEFAULT_PLATFORM=$(PLATFORMS) docker compose --profile $(PLATFORMS) -f docker-compose-v3-test-parallel.yml up -d --remove-orphans --no-log-prefix ; \ RUN_IN_DOCKER_COMPOSE=true bash ./bootstrap.sh $$node ; \ done ; \ docker compose -f docker-compose-v3-test-parallel.yml down +test_video_standalone: standalone_chrome standalone_chromium standalone_firefox standalone_edge + DOCKER_COMPOSE_FILE=docker-compose-v3-test-standalone.yml make test_video + test_video_dynamic_name: - VIDEO_FILE_NAME=auto TEST_DELAY_AFTER_TEST=0 \ + VIDEO_FILE_NAME=auto \ make test_video # This should run on its own CI job. There is no need to combine it with the other tests. # Its main purpose is to check that a video file was generated. test_video: video hub chrome firefox edge chromium sudo rm -rf ./tests/tests - sudo rm -rf ./tests/videos; mkdir -p ./tests/videos + sudo rm -rf ./tests/videos; mkdir -p ./tests/videos/upload + sudo chmod -R 777 ./tests/videos + docker_compose_file=$(or $(DOCKER_COMPOSE_FILE), docker-compose-v3-test-video.yml) ; \ + list_of_tests_amd64=$(or $(LIST_OF_TESTS_AMD64), "NodeChrome NodeChromium NodeFirefox NodeEdge") ; \ + list_of_tests_arm64=$(or $(LIST_OF_TESTS_ARM64), "NodeChromium NodeFirefox") ; \ if [ "$(PLATFORMS)" = "linux/amd64" ]; then \ - list_nodes="NodeChrome NodeChromium NodeFirefox NodeEdge" ; \ + list_nodes="$${list_of_tests_amd64}" ; \ else \ - list_nodes="NodeChromium NodeFirefox" ; \ + list_nodes="${list_of_tests_arm64}" ; \ fi; \ for node in $${list_nodes}; do \ cd ./tests || true ; \ @@ -607,6 +616,10 @@ test_video: video hub chrome firefox edge chromium echo UID=$$(id -u) >> .env ; \ echo BINDING_VERSION=$(BINDING_VERSION) >> .env ; \ echo TEST_DELAY_AFTER_TEST=$(or $(TEST_DELAY_AFTER_TEST), 0) >> .env ; \ + echo SELENIUM_ENABLE_MANAGED_DOWNLOADS=$(or $(SELENIUM_ENABLE_MANAGED_DOWNLOADS), "true") >> .env ; \ + echo BASIC_AUTH_USERNAME=$(or $(BASIC_AUTH_USERNAME), "admin") >> .env ; \ + echo BASIC_AUTH_PASSWORD=$(or $(BASIC_AUTH_PASSWORD), "admin") >> .env ; \ + echo SUB_PATH=$(or $(SUB_PATH), "/selenium") >> .env ; \ if [ $$node = "NodeChrome" ] ; then \ echo BROWSER=chrome >> .env ; \ echo VIDEO_FILE_NAME=$${VIDEO_FILE_NAME:-"chrome_video.mp4"} >> .env ; \ @@ -627,7 +640,7 @@ test_video: video hub chrome firefox edge chromium echo VIDEO_FILE_NAME=$${VIDEO_FILE_NAME:-"firefox_video.mp4"} >> .env ; \ echo VIDEO_FILE_NAME_SUFFIX=$${VIDEO_FILE_NAME_SUFFIX:-"true"} >> .env ; \ fi ; \ - DOCKER_DEFAULT_PLATFORM=$(PLATFORMS) docker compose -f docker-compose-v3-test-video.yml up --abort-on-container-exit ; \ + DOCKER_DEFAULT_PLATFORM=$(PLATFORMS) docker compose -f $${docker_compose_file} up --remove-orphans --build --exit-code-from tests ; \ done make test_video_integrity @@ -676,18 +689,27 @@ test_node_relay: hub node_base standalone_firefox fi ; \ export $$(cat .env | xargs) ; \ envsubst < relay_config.toml > ./videos/relay_config.toml ; \ - DOCKER_DEFAULT_PLATFORM=$(PLATFORMS) docker compose --profile $$node -f docker-compose-v3-test-node-relay.yml up --no-log-prefix --exit-code-from tests ; \ + DOCKER_DEFAULT_PLATFORM=$(PLATFORMS) docker compose --profile $$node -f docker-compose-v3-test-node-relay.yml up --remove-orphans --no-log-prefix --build --exit-code-from tests ; \ if [ $$? -ne 0 ]; then exit 1; fi ; \ done +test_standalone_docker: standalone_docker + DOCKER_COMPOSE_FILE=docker-compose-v3-test-standalone-docker.yaml CONFIG_FILE=standalone_docker_config.toml \ + RECORD_STANDALONE=true GRID_URL=http://0.0.0.0:4444 LIST_OF_TESTS_AMD64="DeploymentAutoscaling" TEST_PARALLEL_HARDENING=true \ + SELENIUM_ENABLE_MANAGED_DOWNLOADS=true LOG_LEVEL=SEVERE SKIP_CHECK_DOWNLOADS_VOLUME=true make test_node_docker + test_node_docker: hub standalone_docker standalone_chrome standalone_firefox standalone_edge standalone_chromium video sudo rm -rf ./tests/tests - sudo rm -rf ./tests/videos; mkdir -p ./tests/videos/Downloads + sudo rm -rf ./tests/videos; mkdir -p ./tests/videos/Downloads; mkdir -p ./tests/videos/upload sudo chmod -R 777 ./tests/videos + docker_compose_file=$(or $(DOCKER_COMPOSE_FILE), docker-compose-v3-test-node-docker.yaml) ; \ + config_file=$(or $(CONFIG_FILE), config.toml) ; \ + list_of_tests_amd64=$(or $(LIST_OF_TESTS_AMD64), "NodeChrome NodeChromium NodeFirefox NodeEdge") ; \ + list_of_tests_arm64=$(or $(LIST_OF_TESTS_ARM64), "NodeChromium NodeFirefox") ; \ if [ "$(PLATFORMS)" = "linux/amd64" ]; then \ - list_nodes="NodeChrome NodeChromium NodeFirefox NodeEdge" ; \ + list_nodes="$${list_of_tests_amd64}" ; \ else \ - list_nodes="NodeChromium NodeFirefox" ; \ + list_nodes="$${list_of_tests_arm64}" ; \ fi; \ for node in $${list_nodes} ; do \ cd tests || true ; \ @@ -702,6 +724,8 @@ test_node_docker: hub standalone_docker standalone_chrome standalone_firefox sta echo REQUEST_TIMEOUT=$(or $(REQUEST_TIMEOUT), 300) >> .env ; \ echo SELENIUM_ENABLE_MANAGED_DOWNLOADS=$(or $(SELENIUM_ENABLE_MANAGED_DOWNLOADS), "false") >> .env ; \ echo TEST_DELAY_AFTER_TEST=$(or $(TEST_DELAY_AFTER_TEST), 0) >> .env ; \ + echo RECORD_STANDALONE=$(or $(RECORD_STANDALONE), "true") >> .env ; \ + echo GRID_URL=$(or $(GRID_URL), "") >> .env ; \ echo NODE=$$node >> .env ; \ echo UID=$$(id -u) >> .env ; \ echo BINDING_VERSION=$(BINDING_VERSION) >> .env ; \ @@ -716,64 +740,20 @@ test_node_docker: hub standalone_docker standalone_chrome standalone_firefox sta fi ; \ if [ $$node = "NodeChromium" ] ; then \ echo NODE_CHROME=chromium >> .env ; \ + else \ + echo NODE_CHROME=chromium >> .env ; \ fi ; \ export $$(cat .env | xargs) ; \ - envsubst < config.toml > ./videos/config.toml ; \ - DOCKER_DEFAULT_PLATFORM=$(PLATFORMS) docker compose -f docker-compose-v3-test-node-docker.yaml up --no-log-prefix --exit-code-from tests ; \ + envsubst < $${config_file} > ./videos/config.toml ; \ + DOCKER_DEFAULT_PLATFORM=$(PLATFORMS) docker compose -f $${docker_compose_file} up --remove-orphans --no-log-prefix --build --exit-code-from tests ; \ if [ $$? -ne 0 ]; then exit 1; fi ; \ - if [ -d "$$DOWNLOADS_DIR" ] && [ $$(ls -1q $$DOWNLOADS_DIR | wc -l) -eq 0 ]; then \ + if [ "$$SKIP_CHECK_DOWNLOADS_VOLUME" != "true" ] && [ -d "$$DOWNLOADS_DIR" ] && [ $$(ls -1q $$DOWNLOADS_DIR | wc -l) -eq 0 ]; then \ echo "Mounted downloads directory is empty. Downloaded files could not be retrieved!" ; \ exit 1 ; \ fi ; \ done make test_video_integrity -test_standalone_docker: standalone_docker standalone_chrome standalone_firefox standalone_edge standalone_chromium video - sudo rm -rf ./tests/tests - sudo rm -rf ./tests/videos; mkdir -p ./tests/videos/Downloads - sudo chmod -R 777 ./tests/videos - if [ "$(PLATFORMS)" = "linux/amd64" ]; then \ - list_nodes="DeploymentAutoscaling" ; \ - else \ - list_nodes="NodeChromium NodeFirefox" ; \ - fi; \ - for node in $${list_nodes} ; do \ - cd tests || true ; \ - DOWNLOADS_DIR="./videos/Downloads" ; \ - sudo rm -rf $$DOWNLOADS_DIR/* ; \ - echo NAMESPACE=$(NAME) > .env ; \ - echo TAG=$(TAG_VERSION) >> .env ; \ - echo VIDEO_TAG=$(FFMPEG_TAG_VERSION)-$(BUILD_DATE) >> .env ; \ - echo TEST_DRAIN_AFTER_SESSION_COUNT=$(or $(TEST_DRAIN_AFTER_SESSION_COUNT), 0) >> .env ; \ - echo TEST_PARALLEL_HARDENING=$(or $(TEST_PARALLEL_HARDENING), "true") >> .env ; \ - echo LOG_LEVEL=$(or $(LOG_LEVEL), "INFO") >> .env ; \ - echo REQUEST_TIMEOUT=$(or $(REQUEST_TIMEOUT), 300) >> .env ; \ - echo SELENIUM_ENABLE_MANAGED_DOWNLOADS=$(or $(SELENIUM_ENABLE_MANAGED_DOWNLOADS), "true") >> .env ; \ - echo TEST_DELAY_AFTER_TEST=$(or $(TEST_DELAY_AFTER_TEST), 0) >> .env ; \ - echo NODE=$$node >> .env ; \ - echo UID=$$(id -u) >> .env ; \ - echo BINDING_VERSION=$(BINDING_VERSION) >> .env ; \ - echo HOST_IP=$$(hostname -I | awk '{print $$1}') >> .env ; \ - if [ "$(PLATFORMS)" = "linux/amd64" ]; then \ - echo NODE_EDGE=edge >> .env ; \ - else \ - echo NODE_EDGE=chromium >> .env ; \ - fi; \ - if [ $$node = "NodeChrome" ] ; then \ - echo NODE_CHROME=chrome >> .env ; \ - fi ; \ - if [ $$node = "NodeChromium" ] ; then \ - echo NODE_CHROME=chromium >> .env ; \ - else \ - echo NODE_CHROME=chromium >> .env ; \ - fi ; \ - export $$(cat .env | xargs) ; \ - envsubst < standalone_docker_config.toml > ./videos/config.toml ; \ - DOCKER_DEFAULT_PLATFORM=$(PLATFORMS) docker compose -f docker-compose-v3-test-standalone-docker.yaml up --no-log-prefix --build --exit-code-from tests ; \ - if [ $$? -ne 0 ]; then exit 1; fi ; \ - done - make test_video_integrity - test_custom_ca_cert: VERSION=$(TAG_VERSION) NAMESPACE=$(NAMESPACE) ./tests/customCACert/bootstrap.sh @@ -800,7 +780,7 @@ test_video_integrity: # Using ffmpeg to verify file integrity # https://superuser.com/questions/100288/how-can-i-check-the-integrity-of-a-video-file-avi-mpeg-mp4 list_files=$$(find ./tests/videos -type f -name "*.mp4"); \ - echo "Number of video files: $$(echo $$list_files | wc -w)"; \ + echo "::warning:: Number of video files: $$(echo $$list_files | wc -w)"; \ number_corrupted_files=0; \ if [ -z "$$list_files" ]; then \ echo "No video files found"; \ @@ -827,8 +807,8 @@ chart_render_template: RENDER_HELM_TEMPLATE_ONLY=true make chart_test_autoscaling_disabled chart_test_autoscaling_deployment_https chart_test_autoscaling_deployment chart_test_autoscaling_job_https chart_test_autoscaling_job_hostname chart_test_autoscaling_job chart_test_autoscaling_disabled: - PLATFORMS=$(PLATFORMS) TEST_CHROMIUM=true RELEASE_NAME=selenium SELENIUM_GRID_AUTOSCALING=false TEST_DELAY_AFTER_TEST=0 CHART_ENABLE_TRACING=true \ - SECURE_INGRESS_ONLY_GENERATE=true SELENIUM_GRID_PROTOCOL=https SELENIUM_GRID_HOST=$$(hostname -i) SELENIUM_GRID_PORT=443 \ + PLATFORMS=$(PLATFORMS) TEST_CHROMIUM=true RELEASE_NAME=selenium SELENIUM_GRID_AUTOSCALING=false CHART_ENABLE_TRACING=true \ + SECURE_INGRESS_ONLY_GENERATE=true SELENIUM_GRID_PROTOCOL=https SELENIUM_GRID_HOST=$$(hostname -i) SELENIUM_GRID_PORT=443 EXTERNAL_UPLOADER_CONFIG=true \ VERSION=$(TAG_VERSION) VIDEO_TAG=$(FFMPEG_TAG_VERSION)-$(BUILD_DATE) NAMESPACE=$(NAMESPACE) BINDING_VERSION=$(BINDING_VERSION) \ TEMPLATE_OUTPUT_FILENAME="k8s_nodeChromium_enableTracing_secureIngress_generateCerts_ingressPublicIP_subPath.yaml" \ ./tests/charts/make/chart_test.sh NoAutoscaling @@ -852,7 +832,7 @@ chart_test_autoscaling_deployment: chart_test_autoscaling_job_https: PLATFORMS=$(PLATFORMS) TEST_EXISTING_KEDA=true RELEASE_NAME=selenium CHART_ENABLE_BASIC_AUTH=true \ SECURE_CONNECTION_SERVER=true SELENIUM_GRID_PROTOCOL=https SELENIUM_GRID_PORT=443 SUB_PATH=/ \ - VERSION=$(TAG_VERSION) VIDEO_TAG=$(FFMPEG_TAG_VERSION)-$(BUILD_DATE) NAMESPACE=$(NAMESPACE) BINDING_VERSION=$(BINDING_VERSION) \ + VERSION=$(TAG_VERSION) VIDEO_TAG=$(FFMPEG_TAG_VERSION)-$(BUILD_DATE) NAMESPACE=$(NAMESPACE) BINDING_VERSION=$(BINDING_VERSION) EXTERNAL_UPLOADER_CONFIG=true \ TEMPLATE_OUTPUT_FILENAME="k8s_prefixSelenium_basicAuth_secureServer_autoScaling_scaledJob_existingKEDA.yaml" \ ./tests/charts/make/chart_test.sh JobAutoscaling diff --git a/NodeFirefox/Dockerfile b/NodeFirefox/Dockerfile index 9bb57d631..1f7167487 100644 --- a/NodeFirefox/Dockerfile +++ b/NodeFirefox/Dockerfile @@ -31,7 +31,7 @@ RUN if [ "$(dpkg --print-architecture)" = "amd64" ]; then \ # GeckoDriver #============ ARG GECKODRIVER_VERSION=latest -RUN LATEST_VERSION=$(curl -s https://api.github.com/repos/mozilla/geckodriver/releases/latest | jq -r '.tag_name') \ +RUN LATEST_VERSION=$(curl -sk https://api.github.com/repos/mozilla/geckodriver/releases/latest | jq -r '.tag_name') \ && DRIVER_ARCH=$(if [ "$(dpkg --print-architecture)" = "amd64" ]; then echo "linux64"; else echo "linux-aarch64"; fi) \ && GK_VERSION=$(if [ ${GECKODRIVER_VERSION:-latest} = "latest" ]; then echo "${LATEST_VERSION}"; else echo $GECKODRIVER_VERSION; fi) \ && echo "Using GeckoDriver version: "$GK_VERSION \ diff --git a/README.md b/README.md index ddf8330dc..35a8d260a 100644 --- a/README.md +++ b/README.md @@ -685,16 +685,30 @@ services: ``` `SE_VIDEO_FILE_NAME=auto` will use the session id as the video file name. This ensures that the video file name is unique to upload. +Video file name construction automatically works based on Node endpoint `/status` (and optional GraphQL endpoint) to get session ID, capabilities. + +| | Hub/Nodes | Standalone roles | Dynamic Grid | +|------------------------------------------|-------------------|------------------|----------------| +| `SE_VIDEO_RECORD_STANDALONE` (mandatory) | `false` (default) | `true` | user input | +| `DISPLAY_CONTAINER_NAME` (mandatory) | user input | user input | (not required) | +| `SE_NODE_PORT` (optional) | `5555` | `4444` | (not required) | +| `SE_NODE_GRID_URL` (optional) | user input | (not required) | (not required) | `SE_VIDEO_UPLOAD_ENABLED=true` will enable the video upload feature. In the background, it will create a pipefile with file and destination for uploader to consume and proceed. `SE_VIDEO_INTERNAL_UPLOAD=true` will use RCLONE installed in the container for upload. If you want to use another container for upload, set it to `false`. For environment variables with prefix `RCLONE_` is used to pass remote configuration to RCLONE. You can find more information about RCLONE configuration [here](https://rclone.org/docs/). +When using in Dynamic Grid, those variables should be combined with the prefix `SE_`, for example `SE_RCLONE_`. See below reference for more details. + +### Reference +- Configure video recording and uploading for Hub and Nodes: [docker-compose-v3-video-upload.yml](docker-compose-v3-video-upload.yml) + +- Configure video recording and uploading for Standalone roles: [docker-compose-v3-video-upload-standalone.yml](docker-compose-v3-video-upload-standalone.yml) -[`docker-compose-v3-video-upload.yml`](docker-compose-v3-video-upload.yml) +- Configure video recording and uploading for Dynamic Grid (node-docker): [docker-compose-v3-video-upload-dynamic-grid.yml](docker-compose-v3-video-upload-dynamic-grid.yml) -Note that upload function is not supported for Dynamic Grid. If you want it, please create a feature request. +- Configure video recording and uploading for Dynamic Grid standalone (standalone-docker): [tests/docker-compose-v3-test-standalone-docker.yaml](tests/docker-compose-v3-test-standalone-docker.yaml) ___ diff --git a/Video/Dockerfile b/Video/Dockerfile index 9a79891e5..53479a7ee 100755 --- a/Video/Dockerfile +++ b/Video/Dockerfile @@ -72,7 +72,7 @@ RUN groupadd ${SEL_GROUP} \ # Add Supervisor configuration files #====================================== COPY supervisord.conf /etc -COPY --chown="${SEL_UID}:${SEL_GID}" entry_point.sh video.sh video_ready.py video_graphQLQuery.sh /opt/bin/ +COPY --chown="${SEL_UID}:${SEL_GID}" entry_point.sh video.sh video_ready.py video_graphQLQuery.sh video_gridUrl.sh /opt/bin/ #====================================== # Add RCLONE for uploading videos diff --git a/Video/supervisord.conf b/Video/supervisord.conf index 145ab95e5..735f82fc0 100755 --- a/Video/supervisord.conf +++ b/Video/supervisord.conf @@ -28,6 +28,8 @@ stdout_logfile_maxbytes=0 [program:video-ready] priority=5 command=python3 /opt/bin/video_ready.py +stopasgroup = true +killasgroup=true autostart=true autorestart=true @@ -38,7 +40,9 @@ stdout_logfile_maxbytes=0 [program:video-upload] priority=10 -command=bash -c "if [ ${SE_VIDEO_INTERNAL_UPLOAD} = "true" ]; then /opt/bin/upload.sh; fi" +command=/opt/bin/upload.sh +stopasgroup = true +killasgroup=true autostart=%(ENV_SE_VIDEO_INTERNAL_UPLOAD)s autorestart=%(ENV_SE_VIDEO_INTERNAL_UPLOAD)s diff --git a/Video/upload.sh b/Video/upload.sh index e294894a4..640b70587 100755 --- a/Video/upload.sh +++ b/Video/upload.sh @@ -9,6 +9,7 @@ UPLOAD_PIPE_FILE_NAME=${UPLOAD_PIPE_FILE_NAME:-"uploadpipe"} SE_VIDEO_INTERNAL_UPLOAD=${SE_VIDEO_INTERNAL_UPLOAD:-"false"} VIDEO_UPLOAD_ENABLED=${VIDEO_UPLOAD_ENABLED:-SE_VIDEO_UPLOAD_ENABLED} SE_VIDEO_UPLOAD_BATCH_CHECK=${SE_VIDEO_UPLOAD_BATCH_CHECK:-"10"} +process_name="video.uploader" if [ "${SE_VIDEO_INTERNAL_UPLOAD}" = "true" ]; then @@ -23,7 +24,7 @@ fi if [ "${UPLOAD_RETAIN_LOCAL_FILE}" = "false" ]; then - echo "UPLOAD_RETAIN_LOCAL_FILE is set to false, force to use RCLONE command: move" + echo "$(date +%FT%T%Z) [${process_name}] - UPLOAD_RETAIN_LOCAL_FILE is set to false, force to use RCLONE command: move" UPLOAD_COMMAND="move" fi @@ -42,14 +43,52 @@ function rename_rclone_env() { done } -function graceful_exit() { - for pid in "${list_rclone_pid[@]}"; +function consume_pipe_file() { + while read FILE DESTINATION < ${UPLOAD_PIPE_FILE}; do - wait ${pid} + if [ "${FILE}" = "exit" ]; + then + FORCE_EXIT=true + exit + elif [ "$FILE" != "" ] && [ "$DESTINATION" != "" ]; + then + rclone_upload "${FILE}" "${DESTINATION}" + elif [ -f ${FORCE_EXIT_FILE} ]; + then + echo "$(date +%FT%T%Z) [${process_name}] - Force exit signal detected" + exit + fi done +} + +list_rclone_pid=() +function check_and_clear_background() { + # Wait for a batch rclone processes to finish + if [ ${#list_rclone_pid[@]} -eq ${SE_VIDEO_UPLOAD_BATCH_CHECK} ]; then + for pid in "${list_rclone_pid[@]}"; + do + wait ${pid} + done + list_rclone_pid=() + fi + +} + +function rclone_upload() { + local source=$1 + local target=$2 + echo "$(date +%FT%T%Z) [${process_name}] - Uploading ${source} to ${target}" + exec rclone --config ${UPLOAD_CONFIG_DIRECTORY}/${UPLOAD_CONFIG_FILE_NAME} ${UPLOAD_COMMAND} --cutoff-mode SOFT --metadata ${UPLOAD_OPTS} "${source}" "${target}" & +} + +function graceful_exit() { + echo "$(date +%FT%T%Z) [${process_name}] - Uploader is shutting down" + if [ "${FORCE_EXIT}" != "true" ]; then + consume_pipe_file + fi + echo "$(date +%FT%T%Z) [${process_name}] - Uploader consumed all files in the pipe" rm -rf ${FORCE_EXIT_FILE} - rm -rf ${UPLOAD_PIPE_FILE} || true - echo "Uploader is ready to shutdown" + echo "$(date +%FT%T%Z) [${process_name}] - Uploader is ready to shutdown" } trap graceful_exit SIGTERM SIGINT EXIT @@ -62,7 +101,7 @@ function create_named_pipe() { rm -f "${UPLOAD_PIPE_FILE}" fi mkfifo "${UPLOAD_PIPE_FILE}" - echo "Created named pipe ${UPLOAD_PIPE_FILE}" + echo "$(date +%FT%T%Z) [${process_name}] - Created named pipe ${UPLOAD_PIPE_FILE}" fi } @@ -76,7 +115,7 @@ while true; do then break else - echo "${UPLOAD_PIPE_FILE} exists but is not a named pipe" + echo "$(date +%FT%T%Z) [${process_name}] - ${UPLOAD_PIPE_FILE} exists but is not a named pipe" create_named_pipe fi else @@ -87,40 +126,15 @@ while true; do ELAPSED_TIME=$((CURRENT_TIME - START_TIME)) if [ ${ELAPSED_TIME} -ge ${TIMEOUT} ]; then - echo "Timeout waiting for ${UPLOAD_PIPE_FILE} to be created" + echo "$(date +%FT%T%Z) [${process_name}] - Timeout waiting for ${UPLOAD_PIPE_FILE} to be created" exit 1 fi - echo "Waiting for ${UPLOAD_PIPE_FILE} to be created" + echo "$(date +%FT%T%Z) [${process_name}] - Waiting for ${UPLOAD_PIPE_FILE} to be created" sleep 1 done -echo "Waiting for video files put into pipe for proceeding to upload" +echo "$(date +%FT%T%Z) [${process_name}] - Waiting for video files put into pipe for proceeding to upload" rename_rclone_env - -list_rclone_pid=() -while read FILE DESTINATION < ${UPLOAD_PIPE_FILE}; -do - if [ "${FILE}" = "exit" ]; - then - exit - elif [ "$FILE" != "" ] && [ "$DESTINATION" != "" ]; - then - echo "Uploading ${FILE} to ${DESTINATION}" - rclone --config ${UPLOAD_CONFIG_DIRECTORY}/${UPLOAD_CONFIG_FILE_NAME} ${UPLOAD_COMMAND} ${UPLOAD_OPTS} "${FILE}" "${DESTINATION}" & - list_rclone_pid+=($!) - else - # Wait for a batch rclone processes to finish - if [ ${#list_rclone_pid[@]} -eq ${SE_VIDEO_UPLOAD_BATCH_CHECK} ]; - then - for pid in "${list_rclone_pid[@]}"; - do - wait ${pid} - done - list_rclone_pid=() - fi - fi -done - -graceful_exit +consume_pipe_file diff --git a/Video/video.sh b/Video/video.sh index 4edc5dae5..63611dd09 100755 --- a/Video/video.sh +++ b/Video/video.sh @@ -14,10 +14,19 @@ UPLOAD_DESTINATION_PREFIX=${UPLOAD_DESTINATION_PREFIX:-$SE_UPLOAD_DESTINATION_PR UPLOAD_PIPE_FILE_NAME=${UPLOAD_PIPE_FILE_NAME:-"uploadpipe"} SE_VIDEO_INTERNAL_UPLOAD=${SE_VIDEO_INTERNAL_UPLOAD:-"false"} SE_SERVER_PROTOCOL=${SE_SERVER_PROTOCOL:-"http"} -SE_NODE_PORT=${SE_NODE_PORT:-"5555"} max_attempts=${SE_VIDEO_WAIT_ATTEMPTS:-50} process_name="video.recorder" +if [ "${SE_VIDEO_RECORD_STANDALONE}" = "true" ]; then + JQ_SESSION_ID_QUERY=".value.nodes[]?.slots[]?.session?.sessionId" + SE_NODE_PORT=${SE_NODE_PORT:-"4444"} + NODE_STATUS_ENDPOINT="$(/opt/bin/video_gridUrl.sh)/status" +else + JQ_SESSION_ID_QUERY=".[]?.node?.slots | .[0]?.session?.sessionId" + SE_NODE_PORT=${SE_NODE_PORT:-"5555"} + NODE_STATUS_ENDPOINT="${SE_SERVER_PROTOCOL}://${DISPLAY_CONTAINER_NAME}:${SE_NODE_PORT}/status" +fi + if [ -d "${VIDEO_FOLDER}" ]; then echo "$(date +%FT%T%Z) [${process_name}] - Video folder exists: ${VIDEO_FOLDER}" @@ -74,6 +83,7 @@ function wait_util_uploader_shutdown() { function send_exit_signal_to_uploader() { if [[ "${VIDEO_UPLOAD_ENABLED}" != "false" ]] && [[ -n "${UPLOAD_DESTINATION_PREFIX}" ]]; then + echo "$(date +%FT%T%Z) [${process_name}] - Sending a signal to force exit the uploader" echo "exit" >> ${UPLOAD_PIPE_FILE} & echo "exit" > ${FORCE_EXIT_FILE} fi @@ -102,10 +112,9 @@ function stop_ffmpeg() { } function stop_recording() { - echo "$(date +%FT%T%Z) [${process_name}] - Stopping to record video" + echo "$(date +%FT%T%Z) [${process_name}] - Recorder is shutting down" stop_ffmpeg recorded_count=$((recorded_count+1)) - recording_started="false" if [[ "${VIDEO_UPLOAD_ENABLED}" != "false" ]] && [[ -n "${UPLOAD_DESTINATION_PREFIX}" ]]; then upload_destination=${UPLOAD_DESTINATION_PREFIX}/${video_file_name} @@ -115,6 +124,7 @@ function stop_recording() { then echo "$(date +%FT%T%Z) [${process_name}] - Upload destination not known since UPLOAD_DESTINATION_PREFIX is not set. Continue without uploading." fi + recording_started="false" } function check_if_recording_inprogress() { @@ -128,7 +138,6 @@ function graceful_exit() { check_if_recording_inprogress send_exit_signal_to_uploader wait_util_uploader_shutdown - rm -rf ${UPLOAD_PIPE_FILE} || true kill -SIGTERM "$(cat /var/run/supervisor/supervisord.pid)" } @@ -164,7 +173,7 @@ else sleep 0.5 attempts=$((attempts+1)) done - if [[ $attempts = "$max_attempts" ]] + if [[ $attempts = "$max_attempts" ]]; then echo "$(date +%FT%T%Z) [${process_name}] - Can not open display, exiting." exit @@ -181,7 +190,7 @@ else recorded_count=0 echo "$(date +%FT%T%Z) [${process_name}] - Checking if node API responds" - until curl --noproxy "*" -sk --request GET ${SE_SERVER_PROTOCOL}://${DISPLAY_CONTAINER_NAME}:${SE_NODE_PORT}/status || [[ $attempts = "$max_attempts" ]] + until curl --noproxy "*" -sk --request GET ${NODE_STATUS_ENDPOINT} || [[ $attempts = "$max_attempts" ]] do if [ $(($attempts % 60)) -eq 0 ]; then @@ -190,15 +199,15 @@ else sleep 0.5 attempts=$((attempts+1)) done - if [[ $attempts = "$max_attempts" ]] + if [[ $attempts = "$max_attempts" ]]; then echo "$(date +%FT%T%Z) [${process_name}] - Can not reach node API, exiting." exit fi - while curl --noproxy "*" -sk --request GET ${SE_SERVER_PROTOCOL}://${DISPLAY_CONTAINER_NAME}:${SE_NODE_PORT}/status > /tmp/status.json + while curl --noproxy "*" -sk --request GET ${NODE_STATUS_ENDPOINT} > /tmp/status.json do - session_id=$(jq -r '.[]?.node?.slots | .[0]?.session?.sessionId' /tmp/status.json) - if [[ "$session_id" != "null" && "$session_id" != "" && "$recording_started" = "false" ]] + session_id=$(jq -r "${JQ_SESSION_ID_QUERY}" /tmp/status.json) + if [[ "$session_id" != "null" && "$session_id" != "" && "$session_id" != "reserved" && "$recording_started" = "false" ]]; then echo "$(date +%FT%T%Z) [${process_name}] - Session: $session_id is created" return_list=($(bash ${VIDEO_CONFIG_DIRECTORY}/video_graphQLQuery.sh "$session_id")) @@ -209,7 +218,7 @@ else jq '.' "/tmp/graphQL_$session_id.json"; fi fi - if [[ "$session_id" != "null" && "$session_id" != "" && "$recording_started" = "false" && "$caps_se_video_record" = "true" ]] + if [[ "$session_id" != "null" && "$session_id" != "" && "$session_id" != "reserved" && "$recording_started" = "false" && "$caps_se_video_record" = "true" ]]; then video_file="${VIDEO_FOLDER}/$video_file_name" echo "$(date +%FT%T%Z) [${process_name}] - Starting to record video" @@ -217,8 +226,8 @@ else -video_size ${VIDEO_SIZE} -r ${FRAME_RATE} -i ${DISPLAY} -codec:v ${CODEC} ${PRESET} -pix_fmt yuv420p "$video_file" & recording_started="true" echo "$(date +%FT%T%Z) [${process_name}] - Video recording started" - sleep 2 - elif [[ "$session_id" != "$prev_session_id" && "$recording_started" = "true" ]] + sleep 1 + elif [[ "$session_id" != "$prev_session_id" && "$recording_started" = "true" ]]; then stop_recording if [[ $max_recorded_count -gt 0 ]] && [[ $recorded_count -ge $max_recorded_count ]]; @@ -226,7 +235,7 @@ else echo "$(date +%FT%T%Z) [${process_name}] - Node will be drained since max sessions reached count number ($max_recorded_count)" exit fi - elif [[ $recording_started = "true" ]] + elif [[ $recording_started = "true" ]]; then echo "$(date +%FT%T%Z) [${process_name}] - Video recording in progress " sleep 1 diff --git a/Video/video_graphQLQuery.sh b/Video/video_graphQLQuery.sh index a7319386e..87c37ff57 100755 --- a/Video/video_graphQLQuery.sh +++ b/Video/video_graphQLQuery.sh @@ -2,17 +2,17 @@ # Define parameters SESSION_ID=$1 -GRAPHQL_ENDPOINT=${2:-$SE_NODE_GRID_GRAPHQL_URL} +if [ -n "${SE_NODE_GRID_GRAPHQL_URL}" ]; then + GRAPHQL_ENDPOINT=${SE_NODE_GRID_GRAPHQL_URL} +else + GRAPHQL_ENDPOINT="$(/opt/bin/video_gridUrl.sh)/graphql" +fi VIDEO_CAP_NAME=${VIDEO_CAP_NAME:-"se:recordVideo"} TEST_NAME_CAP=${TEST_NAME_CAP:-"se:name"} VIDEO_NAME_CAP=${VIDEO_NAME_CAP:-"se:videoName"} VIDEO_FILE_NAME_TRIM=${SE_VIDEO_FILE_NAME_TRIM_REGEX:-"[:alnum:]-_"} VIDEO_FILE_NAME_SUFFIX=${SE_VIDEO_FILE_NAME_SUFFIX:-"true"} -if [ -z "${GRAPHQL_ENDPOINT}" ] && [ -n "${SE_NODE_GRID_URL}" ]; then - GRAPHQL_ENDPOINT="${SE_NODE_GRID_URL}/graphql" -fi - if [ -n "${GRAPHQL_ENDPOINT}" ]; then while true; do # Send GraphQL query diff --git a/Video/video_gridUrl.sh b/Video/video_gridUrl.sh new file mode 100755 index 000000000..f9cbd0028 --- /dev/null +++ b/Video/video_gridUrl.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +max_time=3 + +if [ -n "${SE_ROUTER_USERNAME}" ] && [ -n "${SE_ROUTER_PASSWORD}" ]; then + BASIC_AUTH="${SE_ROUTER_USERNAME}:${SE_ROUTER_PASSWORD}@" +fi +if [ "${SE_SUB_PATH}" = "/" ]; then + SE_SUB_PATH="" +fi + +grid_url="${SE_NODE_GRID_URL}" +if [ -n "${SE_HUB_HOST:-$SE_ROUTER_HOST}" ] && [ -n "${SE_HUB_PORT:-$SE_ROUTER_PORT}" ]; then + grid_url=${SE_SERVER_PROTOCOL}://${BASIC_AUTH}${SE_HUB_HOST:-$SE_ROUTER_HOST}:${SE_HUB_PORT:-$SE_ROUTER_PORT}${SE_SUB_PATH} +elif [ -n "${DISPLAY_CONTAINER_NAME}" ] && [ "${SE_VIDEO_RECORD_STANDALONE}" = "true" ]; then + grid_url="${SE_SERVER_PROTOCOL}://${BASIC_AUTH}${DISPLAY_CONTAINER_NAME}:${SE_NODE_PORT:-4444}${SE_SUB_PATH}" # For standalone mode +fi + +if [[ ${grid_url} == */ ]]; then + grid_url="${grid_url%/}" +fi + +echo "${grid_url}" diff --git a/Video/video_ready.py b/Video/video_ready.py index 5256bb5ff..eee1f42a8 100755 --- a/Video/video_ready.py +++ b/Video/video_ready.py @@ -8,7 +8,10 @@ class Handler(BaseHTTPRequestHandler): def do_GET(self): - video_ready = "ffmpeg" in (p.name().lower() for p in psutil.process_iter()) + if environ.get('SE_VIDEO_UPLOAD_ENABLED', 'false').lower() != 'true' and environ.get('SE_VIDEO_FILE_NAME', 'video.mp4').lower() != 'auto': + video_ready = "ffmpeg" in (p.name().lower() for p in psutil.process_iter()) + else: + video_ready = True response_code = 200 if video_ready else 404 response_text = "ready" if video_ready else "not ready" self.send_response(response_code) diff --git a/docker-compose-v3-video-upload-dynamic-grid.yml b/docker-compose-v3-video-upload-dynamic-grid.yml new file mode 100644 index 000000000..67a27db9e --- /dev/null +++ b/docker-compose-v3-video-upload-dynamic-grid.yml @@ -0,0 +1,50 @@ +# To execute this docker compose yml file use `docker compose -f docker-compose-v3-video-upload-dynamic-grid.yml up` +# Add the `-d` flag at the end for detached execution +# To stop the execution, hit Ctrl+C, and then `docker compose -f docker-compose-v3-video-upload-dynamic-grid.yml down` +version: "3" +services: + # Start a local FTP server to demonstrate video upload with RCLONE (https://github.com/delfer/docker-alpine-ftp-server) + ftp_server: + image: delfer/alpine-ftp-server:latest + container_name: ftp_server + environment: + - USERS=seluser|selenium.dev + volumes: + # Mount the local directory `/tmp/upload` to the FTP server's `/ftp/seluser` directory to check out the uploaded videos + - /tmp/upload:/ftp/seluser + stop_grace_period: 30s + + node-docker: + image: selenium/node-docker:latest + volumes: + - ./assets:/opt/selenium/assets + - ./NodeDocker/config.toml:/opt/selenium/config.toml + - /var/run/docker.sock:/var/run/docker.sock + depends_on: + - selenium-hub + environment: + - SE_EVENT_BUS_HOST=selenium-hub + - SE_EVENT_BUS_PUBLISH_PORT=4442 + - SE_EVENT_BUS_SUBSCRIBE_PORT=4443 + - SE_VIDEO_RECORD_STANDALONE=true + - SE_VIDEO_FILE_NAME=auto + - SE_VIDEO_FILE_NAME_SUFFIX=true + - SE_VIDEO_UPLOAD_ENABLED=true + - SE_VIDEO_INTERNAL_UPLOAD=true + - SE_UPLOAD_DESTINATION_PREFIX=myftp://ftp/seluser + # All configs required for RCLONE to upload to remote name myftp with prefix SE_ + - SE_RCLONE_CONFIG_MYFTP_TYPE=ftp + - SE_RCLONE_CONFIG_MYFTP_HOST=ftp_server + - SE_RCLONE_CONFIG_MYFTP_PORT=21 + - SE_RCLONE_CONFIG_MYFTP_USER=seluser + # Password encrypted using command: rclone obscure + - SE_RCLONE_CONFIG_MYFTP_PASS=KkK8RsUIba-MMTBUSnuYIdAKvcnFyLl2pdhQig + - SE_RCLONE_CONFIG_MYFTP_FTP_CONCURRENCY=10 + + selenium-hub: + image: selenium/hub:latest + container_name: selenium-hub + ports: + - "4442:4442" + - "4443:4443" + - "4444:4444" diff --git a/docker-compose-v3-video-upload-standalone.yml b/docker-compose-v3-video-upload-standalone.yml new file mode 100644 index 000000000..a6b128fbe --- /dev/null +++ b/docker-compose-v3-video-upload-standalone.yml @@ -0,0 +1,120 @@ +# To execute this docker compose yml file use `docker compose -f docker-compose-v3-video-upload-standalone.yml up` +# Add the `-d` flag at the end for detached execution +# To stop the execution, hit Ctrl+C, and then `docker compose -f docker-compose-v3-video-upload-standalone.yml down` +# ${variable_pattern} get value from .env in the same directory +version: "3" +services: + # Start a local FTP server to demonstrate video upload with RCLONE (https://github.com/delfer/docker-alpine-ftp-server) + ftp_server: + image: delfer/alpine-ftp-server:latest + environment: + - USERS=seluser|selenium.dev + volumes: + # Mount the local directory `/home/${USER}/Videos/upload` to the FTP server's `/ftp/seluser` directory to check out the uploaded videos + - /tmp/upload:/ftp/seluser + stop_grace_period: 30s + + standalone_chrome: + image: selenium/standalone-chrome:latest + shm_size: 2gb + ports: + - "4444:4444" + environment: + - SE_ROUTER_USERNAME=admin + - SE_ROUTER_PASSWORD=admin + - SE_SUB_PATH=/selenium + + standalone_edge: + image: selenium/standalone-edge:latest + shm_size: 2gb + ports: + - "5444:4444" + environment: + - SE_ROUTER_USERNAME=admin + - SE_ROUTER_PASSWORD=admin + - SE_SUB_PATH=/selenium + + standalone_firefox: + image: selenium/standalone-firefox:latest + shm_size: 2gb + ports: + - "6444:4444" + environment: + - SE_ROUTER_USERNAME=admin + - SE_ROUTER_PASSWORD=admin + - SE_SUB_PATH=/selenium + + chrome_video: + image: selenium/video:latest + depends_on: + - standalone_chrome + environment: + - SE_ROUTER_USERNAME=admin + - SE_ROUTER_PASSWORD=admin + - SE_SUB_PATH=/selenium + - SE_VIDEO_RECORD_STANDALONE=true + - DISPLAY_CONTAINER_NAME=standalone_chrome + - SE_VIDEO_FILE_NAME=auto + - SE_VIDEO_UPLOAD_ENABLED=true + - SE_VIDEO_INTERNAL_UPLOAD=true + # Remote name and destination path to upload + - SE_UPLOAD_DESTINATION_PREFIX=myftp://ftp/seluser + # All configs required for RCLONE to upload to remote name myftp + - RCLONE_CONFIG_MYFTP_TYPE=ftp + - RCLONE_CONFIG_MYFTP_HOST=ftp_server + - RCLONE_CONFIG_MYFTP_PORT=21 + - RCLONE_CONFIG_MYFTP_USER=seluser + # Password encrypted using command: rclone obscure + - RCLONE_CONFIG_MYFTP_PASS=KkK8RsUIba-MMTBUSnuYIdAKvcnFyLl2pdhQig + - RCLONE_CONFIG_MYFTP_FTP_CONCURRENCY=10 + stop_grace_period: 30s + + edge_video: + image: selenium/video:latest + depends_on: + - standalone_edge + environment: + - SE_ROUTER_USERNAME=admin + - SE_ROUTER_PASSWORD=admin + - SE_SUB_PATH=/selenium + - SE_VIDEO_RECORD_STANDALONE=true + - DISPLAY_CONTAINER_NAME=standalone_edge + - SE_VIDEO_FILE_NAME=auto + - SE_VIDEO_UPLOAD_ENABLED=true + - SE_VIDEO_INTERNAL_UPLOAD=true + # Remote name and destination path to upload + - SE_UPLOAD_DESTINATION_PREFIX=myftp://ftp/seluser + # All configs required for RCLONE to upload to remote name myftp + - RCLONE_CONFIG_MYFTP_TYPE=ftp + - RCLONE_CONFIG_MYFTP_HOST=ftp_server + - RCLONE_CONFIG_MYFTP_PORT=21 + - RCLONE_CONFIG_MYFTP_USER=seluser + # Password encrypted using command: rclone obscure + - RCLONE_CONFIG_MYFTP_PASS=KkK8RsUIba-MMTBUSnuYIdAKvcnFyLl2pdhQig + - RCLONE_CONFIG_MYFTP_FTP_CONCURRENCY=10 + stop_grace_period: 30s + + firefox_video: + image: selenium/video:latest + depends_on: + - standalone_firefox + environment: + - SE_ROUTER_USERNAME=admin + - SE_ROUTER_PASSWORD=admin + - SE_SUB_PATH=/selenium + - SE_VIDEO_RECORD_STANDALONE=true + - DISPLAY_CONTAINER_NAME=standalone_firefox + - SE_VIDEO_FILE_NAME=auto + - SE_VIDEO_UPLOAD_ENABLED=true + - SE_VIDEO_INTERNAL_UPLOAD=true + # Remote name and destination path to upload + - SE_UPLOAD_DESTINATION_PREFIX=myftp://ftp/seluser + # All configs required for RCLONE to upload to remote name myftp + - RCLONE_CONFIG_MYFTP_TYPE=ftp + - RCLONE_CONFIG_MYFTP_HOST=ftp_server + - RCLONE_CONFIG_MYFTP_PORT=21 + - RCLONE_CONFIG_MYFTP_USER=seluser + # Password encrypted using command: rclone obscure + - RCLONE_CONFIG_MYFTP_PASS=KkK8RsUIba-MMTBUSnuYIdAKvcnFyLl2pdhQig + - RCLONE_CONFIG_MYFTP_FTP_CONCURRENCY=10 + stop_grace_period: 30s diff --git a/docker-compose-v3-video-upload.yml b/docker-compose-v3-video-upload.yml index dd6fdcf49..24e2f0bf3 100644 --- a/docker-compose-v3-video-upload.yml +++ b/docker-compose-v3-video-upload.yml @@ -4,6 +4,17 @@ # ${variable_pattern} get value from .env in the same directory version: "3" services: + # Start a local FTP server to demonstrate video upload with RCLONE (https://github.com/delfer/docker-alpine-ftp-server) + ftp_server: + image: delfer/alpine-ftp-server:latest + container_name: ftp_server + environment: + - USERS=seluser|selenium.dev + volumes: + # Mount the local directory `/tmp/upload` to the FTP server's `/ftp/seluser` directory to check out the uploaded videos + - /tmp/upload:/ftp/seluser + stop_grace_period: 30s + chrome: image: selenium/node-chrome:4.23.1-20240813 shm_size: 2gb @@ -44,17 +55,16 @@ services: - SE_VIDEO_FILE_NAME=auto - SE_VIDEO_UPLOAD_ENABLED=true - SE_VIDEO_INTERNAL_UPLOAD=true - - SE_UPLOAD_DESTINATION_PREFIX=s3://${BUCKET_NAME} - - RCLONE_CONFIG_S3_TYPE=s3 - - RCLONE_CONFIG_S3_PROVIDER=GCS - - RCLONE_CONFIG_S3_ENV_AUTH=true - - RCLONE_CONFIG_S3_REGION=asia-southeast1 - - RCLONE_CONFIG_S3_LOCATION_CONSTRAINT=asia-southeast1 - - RCLONE_CONFIG_S3_ACL=private - - RCLONE_CONFIG_S3_ACCESS_KEY_ID=${GS_ACCESS_KEY_ID} - - RCLONE_CONFIG_S3_SECRET_ACCESS_KEY=${GS_SECRET_ACCESS_KEY} - - RCLONE_CONFIG_S3_ENDPOINT=https://storage.googleapis.com - - RCLONE_CONFIG_S3_NO_CHECK_BUCKET=true + # Remote name and destination path to upload + - SE_UPLOAD_DESTINATION_PREFIX=myftp://ftp/seluser + # All configs required for RCLONE to upload to remote name myftp + - RCLONE_CONFIG_MYFTP_TYPE=ftp + - RCLONE_CONFIG_MYFTP_HOST=ftp_server + - RCLONE_CONFIG_MYFTP_PORT=21 + - RCLONE_CONFIG_MYFTP_USER=seluser + # Password encrypted using command: rclone obscure + - RCLONE_CONFIG_MYFTP_PASS=KkK8RsUIba-MMTBUSnuYIdAKvcnFyLl2pdhQig + - RCLONE_CONFIG_MYFTP_FTP_CONCURRENCY=10 edge_video: image: selenium/video:ffmpeg-7.0.1-20240813 @@ -66,17 +76,16 @@ services: - SE_VIDEO_FILE_NAME=auto - SE_VIDEO_UPLOAD_ENABLED=true - SE_VIDEO_INTERNAL_UPLOAD=true - - SE_UPLOAD_DESTINATION_PREFIX=s3://${BUCKET_NAME} - - RCLONE_CONFIG_S3_TYPE=s3 - - RCLONE_CONFIG_S3_PROVIDER=GCS - - RCLONE_CONFIG_S3_ENV_AUTH=true - - RCLONE_CONFIG_S3_REGION=asia-southeast1 - - RCLONE_CONFIG_S3_LOCATION_CONSTRAINT=asia-southeast1 - - RCLONE_CONFIG_S3_ACL=private - - RCLONE_CONFIG_S3_ACCESS_KEY_ID=${GS_ACCESS_KEY_ID} - - RCLONE_CONFIG_S3_SECRET_ACCESS_KEY=${GS_SECRET_ACCESS_KEY} - - RCLONE_CONFIG_S3_ENDPOINT=https://storage.googleapis.com - - RCLONE_CONFIG_S3_NO_CHECK_BUCKET=true + # Remote name and destination path to upload + - SE_UPLOAD_DESTINATION_PREFIX=myftp://ftp/seluser + # All configs required for RCLONE to upload to remote name myftp + - RCLONE_CONFIG_MYFTP_TYPE=ftp + - RCLONE_CONFIG_MYFTP_HOST=ftp_server + - RCLONE_CONFIG_MYFTP_PORT=21 + - RCLONE_CONFIG_MYFTP_USER=seluser + # Password encrypted using command: rclone obscure + - RCLONE_CONFIG_MYFTP_PASS=KkK8RsUIba-MMTBUSnuYIdAKvcnFyLl2pdhQig + - RCLONE_CONFIG_MYFTP_FTP_CONCURRENCY=10 firefox_video: image: selenium/video:ffmpeg-7.0.1-20240813 @@ -88,17 +97,16 @@ services: - SE_VIDEO_FILE_NAME=auto - SE_VIDEO_UPLOAD_ENABLED=true - SE_VIDEO_INTERNAL_UPLOAD=true - - SE_UPLOAD_DESTINATION_PREFIX=s3://${BUCKET_NAME} - - RCLONE_CONFIG_S3_TYPE=s3 - - RCLONE_CONFIG_S3_PROVIDER=GCS - - RCLONE_CONFIG_S3_ENV_AUTH=true - - RCLONE_CONFIG_S3_REGION=asia-southeast1 - - RCLONE_CONFIG_S3_LOCATION_CONSTRAINT=asia-southeast1 - - RCLONE_CONFIG_S3_ACL=private - - RCLONE_CONFIG_S3_ACCESS_KEY_ID=${GS_ACCESS_KEY_ID} - - RCLONE_CONFIG_S3_SECRET_ACCESS_KEY=${GS_SECRET_ACCESS_KEY} - - RCLONE_CONFIG_S3_ENDPOINT=https://storage.googleapis.com - - RCLONE_CONFIG_S3_NO_CHECK_BUCKET=true + # Remote name and destination path to upload + - SE_UPLOAD_DESTINATION_PREFIX=myftp://ftp/seluser + # All configs required for RCLONE to upload to remote name myftp + - RCLONE_CONFIG_MYFTP_TYPE=ftp + - RCLONE_CONFIG_MYFTP_HOST=ftp_server + - RCLONE_CONFIG_MYFTP_PORT=21 + - RCLONE_CONFIG_MYFTP_USER=seluser + # Password encrypted using command: rclone obscure + - RCLONE_CONFIG_MYFTP_PASS=KkK8RsUIba-MMTBUSnuYIdAKvcnFyLl2pdhQig + - RCLONE_CONFIG_MYFTP_FTP_CONCURRENCY=10 selenium-hub: image: selenium/hub:4.23.1-20240813 diff --git a/tests/bootstrap.sh b/tests/bootstrap.sh index d41852272..0b1f0b48f 100755 --- a/tests/bootstrap.sh +++ b/tests/bootstrap.sh @@ -24,4 +24,6 @@ if [ "${CI:-false}" = "false" ]; then deactivate fi +sleep 5 + exit $ret_code diff --git a/tests/charts/ci/base-recorder-values.yaml b/tests/charts/ci/base-recorder-values.yaml index b9d11c36d..4e75424ba 100644 --- a/tests/charts/ci/base-recorder-values.yaml +++ b/tests/charts/ci/base-recorder-values.yaml @@ -13,29 +13,16 @@ videoRecorder: enabled: true extraVolumes: - - name: videos - persistentVolumeClaim: - claimName: ${TEST_PV_CLAIM_NAME} +# - name: videos +# persistentVolumeClaim: +# claimName: ${TEST_PV_CLAIM_NAME} uploader: - enabled: "${UPLOAD_ENABLED}" - destinationPrefix: "gs://${BUCKET_NAME}" + enabled: true + destinationPrefix: "myftp://ftp/seluser" secrets: - RCLONE_CONFIG_S3_TYPE: "s3" - RCLONE_CONFIG_S3_PROVIDER: "AWS" - RCLONE_CONFIG_S3_ENV_AUTH: "true" - RCLONE_CONFIG_S3_REGION: "ap-southeast-1" - RCLONE_CONFIG_S3_LOCATION_CONSTRAINT: "ap-southeast-1" - RCLONE_CONFIG_S3_ACL: "private" - RCLONE_CONFIG_S3_ACCESS_KEY_ID: "${AWS_ACCESS_KEY_ID}" - RCLONE_CONFIG_S3_SECRET_ACCESS_KEY: "${AWS_SECRET_ACCESS_KEY}" - RCLONE_CONFIG_S3_NO_CHECK_BUCKET: "true" - RCLONE_CONFIG_GS_TYPE: "s3" - RCLONE_CONFIG_GS_PROVIDER: "GCS" - RCLONE_CONFIG_GS_ENV_AUTH: "true" - RCLONE_CONFIG_GS_REGION: "asia-southeast1" - RCLONE_CONFIG_GS_LOCATION_CONSTRAINT: "asia-southeast1" - RCLONE_CONFIG_GS_ACL: "private" - RCLONE_CONFIG_GS_ACCESS_KEY_ID: "${GS_ACCESS_KEY_ID}" - RCLONE_CONFIG_GS_SECRET_ACCESS_KEY: "${GS_SECRET_ACCESS_KEY}" - RCLONE_CONFIG_GS_ENDPOINT: "https://storage.googleapis.com" - RCLONE_CONFIG_GS_NO_CHECK_BUCKET: "true" + RCLONE_CONFIG_MYFTP_TYPE: "ftp" + RCLONE_CONFIG_MYFTP_HOST: "ftp-server" + RCLONE_CONFIG_MYFTP_PORT: "21" + RCLONE_CONFIG_MYFTP_USER: "seluser" + RCLONE_CONFIG_MYFTP_PASS: "KkK8RsUIba-MMTBUSnuYIdAKvcnFyLl2pdhQig" + RCLONE_CONFIG_MYFTP_FTP_CONCURRENCY: "5" diff --git a/tests/charts/ci/local-pvc.yaml b/tests/charts/ci/local-pvc.yaml index b506e9647..a02e42418 100644 --- a/tests/charts/ci/local-pvc.yaml +++ b/tests/charts/ci/local-pvc.yaml @@ -1,7 +1,81 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: ftp-server + namespace: ${SELENIUM_NAMESPACE} +spec: + replicas: 1 + selector: + matchLabels: + app: ftp-server + template: + metadata: + labels: + app: ftp-server + spec: + containers: + - name: ftp-server + image: delfer/alpine-ftp-server:latest + env: + - name: USERS + value: "seluser|selenium.dev" + - name: MAX_PORT + value: "21005" + volumeMounts: + - mountPath: /ftp/seluser + name: ftp-upload + subPath: seluser + volumes: + - name: ftp-upload + persistentVolumeClaim: + claimName: ${TEST_PV_CLAIM_NAME} +--- +apiVersion: v1 +kind: Service +metadata: + name: ftp-server + namespace: ${SELENIUM_NAMESPACE} + labels: + app: ftp-server +spec: + selector: + app: ftp-server + ports: + - protocol: TCP + name: ftp + port: 21 + targetPort: 21 + - protocol: TCP + name: ftp-data0 + port: 21000 + targetPort: 21000 + - protocol: TCP + name: ftp-data1 + port: 21001 + targetPort: 21001 + - protocol: TCP + name: ftp-data2 + port: 21002 + targetPort: 21002 + - protocol: TCP + name: ftp-data3 + port: 21003 + targetPort: 21003 + - protocol: TCP + name: ftp-data4 + port: 21004 + targetPort: 21004 + - protocol: TCP + name: ftp-data5 + port: 21005 + targetPort: 21005 +--- apiVersion: v1 kind: PersistentVolume metadata: name: pv-local + labels: + app: ftp-server spec: accessModes: - "ReadWriteOnce" @@ -21,6 +95,8 @@ kind: PersistentVolumeClaim metadata: name: ${TEST_PV_CLAIM_NAME} namespace: ${SELENIUM_NAMESPACE} + labels: + app: ftp-server spec: accessModes: - "ReadWriteOnce" diff --git a/tests/charts/ci/uploader.conf b/tests/charts/ci/uploader.conf new file mode 100644 index 000000000..72eff859b --- /dev/null +++ b/tests/charts/ci/uploader.conf @@ -0,0 +1,7 @@ +[myftp] +type = ftp +host = ftp-server +port = 21 +user = seluser +pass = KkK8RsUIba-MMTBUSnuYIdAKvcnFyLl2pdhQig +ftp-concurrency = 5 diff --git a/tests/charts/make/chart_test.sh b/tests/charts/make/chart_test.sh index c74746af0..2e1c1f99f 100755 --- a/tests/charts/make/chart_test.sh +++ b/tests/charts/make/chart_test.sh @@ -88,8 +88,6 @@ fi if [ -f .env ] then export "$(cat .env | xargs)" -else - export UPLOAD_ENABLED=false fi export RELEASE_NAME=${RELEASE_NAME} export SELENIUM_NAMESPACE=${SELENIUM_NAMESPACE} @@ -107,7 +105,7 @@ if [ "${TEST_UPGRADE_CHART}" != "true" ] && [ "${RENDER_HELM_TEMPLATE_ONLY}" != sudo chmod -R 777 ${HOST_PATH} kubectl create ns ${SELENIUM_NAMESPACE} || true kubectl apply -n ${SELENIUM_NAMESPACE} -f ${LOCAL_PVC_YAML} - kubectl describe pv,pvc -n ${SELENIUM_NAMESPACE} + kubectl describe pod,svc,pv,pvc -n ${SELENIUM_NAMESPACE} -l app=ftp-server fi HELM_COMMAND_SET_IMAGES=" \ @@ -271,6 +269,13 @@ if [ "${SELENIUM_GRID_AUTOSCALING}" = "true" ]; then " fi +if [ "${EXTERNAL_UPLOADER_CONFIG}" = "true" ]; then + HELM_COMMAND_SET_IMAGES="${HELM_COMMAND_SET_IMAGES} \ + --set videoRecorder.uploader.secrets=null \ + --set-file uploaderConfigMap.secretFiles.upload\.conf=${TEST_VALUES_PATH}/uploader.conf \ + " +fi + HELM_COMMAND_SET_BASE_VALUES=" \ --values ${TEST_VALUES_PATH}/base-auth-ingress-values.yaml \ --values ${RECORDER_VALUES_FILE} \ diff --git a/tests/docker-compose-v3-test-node-docker.yaml b/tests/docker-compose-v3-test-node-docker.yaml index f93fb592d..c77a7134e 100644 --- a/tests/docker-compose-v3-test-node-docker.yaml +++ b/tests/docker-compose-v3-test-node-docker.yaml @@ -24,6 +24,18 @@ services: - SE_OPTS=--enable-managed-downloads ${SELENIUM_ENABLE_MANAGED_DOWNLOADS} - SE_BROWSER_ARGS_DISABLE_DSHM=--disable-dev-shm-usage - SE_LOG_LEVEL=${LOG_LEVEL} + - SE_VIDEO_RECORD_STANDALONE=true + - SE_VIDEO_FILE_NAME=${VIDEO_FILE_NAME} + - SE_VIDEO_FILE_NAME_SUFFIX=${VIDEO_FILE_NAME_SUFFIX} + - SE_VIDEO_UPLOAD_ENABLED=true + - SE_VIDEO_INTERNAL_UPLOAD=true + - SE_UPLOAD_DESTINATION_PREFIX=myftp://ftp/seluser + - SE_RCLONE_CONFIG_MYFTP_TYPE=ftp + - SE_RCLONE_CONFIG_MYFTP_HOST=ftp_server + - SE_RCLONE_CONFIG_MYFTP_PORT=21 + - SE_RCLONE_CONFIG_MYFTP_USER=seluser + - SE_RCLONE_CONFIG_MYFTP_PASS=KkK8RsUIba-MMTBUSnuYIdAKvcnFyLl2pdhQig + - SE_RCLONE_CONFIG_MYFTP_FTP_CONCURRENCY=10 selenium-hub: image: ${NAMESPACE}/hub:${TAG} @@ -37,6 +49,16 @@ services: - "4443:4443" - "4444:4444" + ftp_server: + image: delfer/alpine-ftp-server:latest + container_name: ftp_server + environment: + - USERS=seluser|selenium.dev + - MAX_PORT=21010 + volumes: + - ./videos/upload:/ftp/seluser + stop_grace_period: 30s + tests: image: docker-selenium-tests:latest build: diff --git a/tests/docker-compose-v3-test-standalone-docker.yaml b/tests/docker-compose-v3-test-standalone-docker.yaml index 81661babe..c6463e9bf 100644 --- a/tests/docker-compose-v3-test-standalone-docker.yaml +++ b/tests/docker-compose-v3-test-standalone-docker.yaml @@ -11,8 +11,20 @@ services: - SE_START_VNC=true - SE_LOG_LEVEL=${LOG_LEVEL} - SE_NODE_ENABLE_MANAGED_DOWNLOADS=${SELENIUM_ENABLE_MANAGED_DOWNLOADS} - - SE_NODE_GRID_URL=http://0.0.0.0:4444 + - SE_NODE_GRID_URL=${GRID_URL} - SE_OPTS=--log-level ${LOG_LEVEL} --enable-managed-downloads ${SELENIUM_ENABLE_MANAGED_DOWNLOADS} + - SE_VIDEO_RECORD_STANDALONE=${RECORD_STANDALONE} + - SE_VIDEO_FILE_NAME=${VIDEO_FILE_NAME} + - SE_VIDEO_FILE_NAME_SUFFIX=${VIDEO_FILE_NAME_SUFFIX} + - SE_VIDEO_UPLOAD_ENABLED=true + - SE_VIDEO_INTERNAL_UPLOAD=true + - SE_UPLOAD_DESTINATION_PREFIX=myftp://ftp/seluser + - SE_RCLONE_CONFIG_MYFTP_TYPE=ftp + - SE_RCLONE_CONFIG_MYFTP_HOST=ftp_server + - SE_RCLONE_CONFIG_MYFTP_PORT=21 + - SE_RCLONE_CONFIG_MYFTP_USER=seluser + - SE_RCLONE_CONFIG_MYFTP_PASS=KkK8RsUIba-MMTBUSnuYIdAKvcnFyLl2pdhQig + - SE_RCLONE_CONFIG_MYFTP_FTP_CONCURRENCY=10 container_name: selenium-hub ports: - "4444:4444" @@ -22,6 +34,16 @@ services: timeout: 30s retries: 5 + ftp_server: + image: delfer/alpine-ftp-server:latest + container_name: ftp_server + environment: + - USERS=seluser|selenium.dev + - MAX_PORT=21010 + volumes: + - ./videos/upload:/ftp/seluser + stop_grace_period: 30s + tests: image: docker-selenium-tests:latest build: diff --git a/tests/docker-compose-v3-test-standalone.yml b/tests/docker-compose-v3-test-standalone.yml new file mode 100644 index 000000000..a51ca1bbd --- /dev/null +++ b/tests/docker-compose-v3-test-standalone.yml @@ -0,0 +1,75 @@ +# How to run this? +# Ensure .env file with following variables is present in the same directory as this file +# docker compose -f docker-compose-v3-test-standalone.yml up --abort-on-container-exit --build +# To clean up, `docker compose -f docker-compose-v3-test-standalone.yml down` +version: "3" +services: + standalone: + image: selenium/standalone-${BROWSER}:${TAG} + user: ${UID} + shm_size: 2gb + container_name: standalone + environment: + - SE_NODE_MAX_SESSIONS=1 + - SE_NODE_OVERRIDE_MAX_SESSIONS=true + - SE_NODE_ENABLE_MANAGED_DOWNLOADS=${SELENIUM_ENABLE_MANAGED_DOWNLOADS} + - SE_OPTS=--enable-managed-downloads ${SELENIUM_ENABLE_MANAGED_DOWNLOADS} + - SE_ROUTER_USERNAME=${BASIC_AUTH_USERNAME} + - SE_ROUTER_PASSWORD=${BASIC_AUTH_PASSWORD} + - SE_SUB_PATH=${SUB_PATH} + ports: + - "4444:4444" + + browser_video: + image: selenium/video:${VIDEO_TAG} + user: ${UID} + depends_on: + - standalone + - ftp_server + environment: + - SE_VIDEO_RECORD_STANDALONE=true + - DISPLAY_CONTAINER_NAME=standalone + - SE_ROUTER_USERNAME=${BASIC_AUTH_USERNAME} + - SE_ROUTER_PASSWORD=${BASIC_AUTH_PASSWORD} + - SE_SUB_PATH=${SUB_PATH} + - SE_VIDEO_FILE_NAME=${VIDEO_FILE_NAME} + - SE_VIDEO_FILE_NAME_SUFFIX=${VIDEO_FILE_NAME_SUFFIX} + - SE_SUPERVISORD_LOG_LEVEL=error + - SE_VIDEO_UPLOAD_ENABLED=true + - SE_VIDEO_INTERNAL_UPLOAD=true + - SE_UPLOAD_DESTINATION_PREFIX=myftp://ftp/seluser + - RCLONE_CONFIG_MYFTP_TYPE=ftp + - RCLONE_CONFIG_MYFTP_HOST=ftp_server + - RCLONE_CONFIG_MYFTP_PORT=21 + - RCLONE_CONFIG_MYFTP_USER=seluser + - RCLONE_CONFIG_MYFTP_PASS=KkK8RsUIba-MMTBUSnuYIdAKvcnFyLl2pdhQig + - RCLONE_CONFIG_MYFTP_FTP_CONCURRENCY=10 + stop_grace_period: 30s + + ftp_server: + image: delfer/alpine-ftp-server:latest + container_name: ftp_server + environment: + - USERS=seluser|selenium.dev + - MAX_PORT=21010 + volumes: + - ./videos/upload:/ftp/seluser + stop_grace_period: 30s + + tests: + image: docker-selenium-tests:latest + build: + context: ./ + dockerfile: ./Dockerfile + depends_on: + - standalone + environment: + - RUN_IN_DOCKER_COMPOSE=true + - SELENIUM_GRID_HOST=standalone + - SELENIUM_GRID_PORT=4444${SUB_PATH} + - SELENIUM_GRID_USERNAME=${BASIC_AUTH_USERNAME} + - SELENIUM_GRID_PASSWORD=${BASIC_AUTH_PASSWORD} + - BINDING_VERSION=${BINDING_VERSION} + - TEST_DELAY_AFTER_TEST=${TEST_DELAY_AFTER_TEST} + - SELENIUM_ENABLE_MANAGED_DOWNLOADS=${SELENIUM_ENABLE_MANAGED_DOWNLOADS} + command: ["/bin/bash", "-c", "./bootstrap.sh ${NODE}"] diff --git a/tests/docker-compose-v3-test-video.yml b/tests/docker-compose-v3-test-video.yml index 73182b49a..2f45b0847 100644 --- a/tests/docker-compose-v3-test-video.yml +++ b/tests/docker-compose-v3-test-video.yml @@ -13,15 +13,13 @@ services: - SE_EVENT_BUS_HOST=selenium-hub - SE_EVENT_BUS_PUBLISH_PORT=4442 - SE_EVENT_BUS_SUBSCRIBE_PORT=4443 - - SE_NODE_ENABLE_MANAGED_DOWNLOADS=true + - SE_NODE_ENABLE_MANAGED_DOWNLOADS=${SELENIUM_ENABLE_MANAGED_DOWNLOADS} ports: - "6900:5900" browser_video: image: selenium/video:${VIDEO_TAG} user: ${UID} - volumes: - - ./videos:/videos depends_on: - browser environment: @@ -30,6 +28,26 @@ services: - SE_VIDEO_FILE_NAME=${VIDEO_FILE_NAME} - SE_VIDEO_FILE_NAME_SUFFIX=${VIDEO_FILE_NAME_SUFFIX} - SE_SUPERVISORD_LOG_LEVEL=error + - SE_VIDEO_UPLOAD_ENABLED=true + - SE_VIDEO_INTERNAL_UPLOAD=true + - SE_UPLOAD_DESTINATION_PREFIX=myftp://ftp/seluser + - RCLONE_CONFIG_MYFTP_TYPE=ftp + - RCLONE_CONFIG_MYFTP_HOST=ftp_server + - RCLONE_CONFIG_MYFTP_PORT=21 + - RCLONE_CONFIG_MYFTP_USER=seluser + - RCLONE_CONFIG_MYFTP_PASS=KkK8RsUIba-MMTBUSnuYIdAKvcnFyLl2pdhQig + - RCLONE_CONFIG_MYFTP_FTP_CONCURRENCY=10 + stop_grace_period: 30s + + ftp_server: + image: delfer/alpine-ftp-server:latest + container_name: ftp_server + environment: + - USERS=seluser|selenium.dev + - MAX_PORT=21010 + volumes: + - ./videos/upload:/ftp/seluser + stop_grace_period: 30s selenium-hub: image: selenium/hub:${TAG} @@ -52,4 +70,5 @@ services: - SELENIUM_GRID_HOST=selenium-hub - BINDING_VERSION=${BINDING_VERSION} - TEST_DELAY_AFTER_TEST=${TEST_DELAY_AFTER_TEST} - command: ["/bin/bash", "-c", "./bootstrap.sh ${NODE} && sleep 5"] + - SELENIUM_ENABLE_MANAGED_DOWNLOADS=${SELENIUM_ENABLE_MANAGED_DOWNLOADS} + command: ["/bin/bash", "-c", "./bootstrap.sh ${NODE}"] diff --git a/tests/standalone_docker_config.toml b/tests/standalone_docker_config.toml index f768a7574..5c1041ba5 100755 --- a/tests/standalone_docker_config.toml +++ b/tests/standalone_docker_config.toml @@ -12,10 +12,10 @@ video-image = "${NAMESPACE}/video:${VIDEO_TAG}" [node] enable-managed-downloads = "${SELENIUM_ENABLE_MANAGED_DOWNLOADS}" override-max-sessions = true -max-sessions = 10 +max-sessions = 16 session-timeout = 1000 enable-cdp = true -selenium-manager = true +selenium-manager = false [sessionqueue] session-request-timeout = "${REQUEST_TIMEOUT}"