Add deps mount:

- Added support for installing CI dependencies via a new environment variable `INSTALL_CI_DEPS`.
- Introduced a new script `install_deps.py` to streamline the installation of pip and npm dependencies based on specified targets.
- Updated `docker-compose.yml` to include a new volume for dependencies and adjusted service configurations accordingly.
- Modified `Makefile-docker` to incorporate checks for CI dependencies and streamline the update process.
- Enhanced GitHub Actions workflows to support the new CI dependency installation feature.

.github/actions/run-docker/action.yml

@@ -23,6 +23,10 @@ inputs:
     description: 'Skip data backup'
     required: false
     default: 'true'
+  install_ci_deps:
+    description: 'Install CI dependencies'
+    required: false
+    default: 'true'
 
 runs:
   using: 'composite'
@@ -40,6 +44,7 @@ runs:
         COMPOSE_FILE: ${{ inputs.compose_file }}
         HOST_UID: ${{ steps.id.outputs.id }}
         DATA_BACKUP_SKIP: ${{ inputs.data_backup_skip }}
+        INSTALL_CI_DEPS: ${{ inputs.install_ci_deps }}
       run: |
         # Start the specified services
         make up

.github/workflows/_test_check.yml

@@ -46,6 +46,7 @@ jobs:
     name: |
       version: '${{ matrix.version }}' |
       compose_file: '${{ matrix.compose_file }}'
+      install_ci_deps: '${{ matrix.install_ci_deps }}'
     strategy:
       fail-fast: false
       matrix:
@@ -55,6 +56,9 @@ jobs:
         compose_file:
           - docker-compose.yml
           - docker-compose.yml:docker-compose.ci.yml
+        install_ci_deps:
+          - false
+          - true
     steps:
       - uses: actions/checkout@v4
       - shell: bash
@@ -63,7 +67,7 @@ jobs:
           cat <<EOF
             Values passed to the action:
             version: ${{ matrix.version }}
-            compose_file: ${{ matrix.compose_file }}
+            install_ci_deps: ${{ matrix.install_ci_deps }}
           EOF
       - uses: ./.github/actions/run-docker
         # Set environment variables that are expected to be ignored
@@ -72,7 +76,7 @@ jobs:
           DOCKER_VERSION: 'not-expected'
         with:
           version: ${{ matrix.version }}
-          compose_file: ${{ matrix.compose_file }}
+          install_ci_deps: ${{ matrix.install_ci_deps }}
           run: make check
 
   test_make_docker_configuration:

Dockerfile

@@ -57,10 +57,6 @@ chown -R olympia:olympia /deps
 ln -s /deps/bin/uwsgi /usr/bin/uwsgi
 ln -s /usr/bin/uwsgi /usr/sbin/uwsgi
 
-# link to the package*.json at ${HOME} so npm can install in /deps
-ln -s ${HOME}/package.json /deps/package.json
-ln -s ${HOME}/package-lock.json /deps/package-lock.json
-
 # Create the storage directory and the test file to verify nginx routing
 mkdir -p ${HOME}/storage
 chown -R olympia:olympia ${HOME}/storage
@@ -86,13 +82,12 @@ ENV NPM_ARGS="--prefix ${NPM_CONFIG_PREFIX} --cache ${NPM_CACHE_DIR} --loglevel
 # All we need in "base" is pip to be installed
 #this let's other layers install packages using the correct version.
 RUN \
+    --mount=type=bind,source=scripts/install_deps.py,target=${HOME}/scripts/install_deps.py \
     # Files required to install pip dependencies
     --mount=type=bind,source=./requirements/pip.txt,target=${HOME}/requirements/pip.txt \
     --mount=type=cache,target=${PIP_CACHE_DIR},uid=${OLYMPIA_UID},gid=${OLYMPIA_UID} \
 <<EOF
-# Work arounds "Multiple .dist-info directories" issue.
-rm -rf /deps/build/*
-${PIP_COMMAND} install --progress-bar=off --no-deps --exists-action=w -r requirements/pip.txt
+${HOME}/scripts/install_deps.py pip
 EOF
 
 # Expose the DOCKER_TARGET variable to all subsequent stages
@@ -106,6 +101,7 @@ ENV DOCKER_TARGET=${DOCKER_TARGET}
 FROM base AS pip_production
 
 RUN \
+    --mount=type=bind,source=scripts/install_deps.py,target=${HOME}/scripts/install_deps.py \
     # Files required to install pip dependencies
     --mount=type=bind,source=./requirements/prod.txt,target=${HOME}/requirements/prod.txt \
     # Files required to install npm dependencies
@@ -115,15 +111,16 @@ RUN \
     --mount=type=cache,target=${PIP_CACHE_DIR},uid=${OLYMPIA_UID},gid=${OLYMPIA_UID} \
     --mount=type=cache,target=${NPM_CACHE_DIR},uid=${OLYMPIA_UID},gid=${OLYMPIA_UID} \
 <<EOF
-${PIP_COMMAND} install --progress-bar=off --no-deps --exists-action=w -r requirements/prod.txt
-npm ci ${NPM_ARGS} --include=prod
+${HOME}/scripts/install_deps.py prod
 EOF
 
 FROM pip_production AS pip_development
 
 RUN \
+    --mount=type=bind,source=scripts/install_deps.py,target=${HOME}/scripts/install_deps.py \
     # Files required to install pip dependencies
     --mount=type=bind,source=./requirements/prod.txt,target=${HOME}/requirements/prod.txt \
+    --mount=type=bind,source=./requirements/ci.txt,target=${HOME}/requirements/ci.txt \
     --mount=type=bind,source=./requirements/dev.txt,target=${HOME}/requirements/dev.txt \
     # Files required to install npm dependencies
     --mount=type=bind,source=package.json,target=${HOME}/package.json \

Makefile-docker

@@ -28,12 +28,16 @@ check_debian_packages: ## check the existence of multiple debian packages
 
 .PHONY: check_pip_packages
 check_pip_packages: ## check the existence of multiple python packages
-	@ ./scripts/check_pip_packages.sh prod.txt
-# "production" corresponds to the "propduction" DOCKER_TARGET defined in the Dockerfile
-# When the target is "production" it means we cannot expect dev.txt dependencies to be installed.
-	@if [ "$(DOCKER_TARGET)" != "production" ]; then \
-		./scripts/check_pip_packages.sh dev.txt; \
-	fi
+	./scripts/check_pip_packages.sh pip.txt
+	./scripts/check_pip_packages.sh prod.txt
+# We expect dev dependencies only if DOCKER_TARGET is not production
+ifeq ($(DOCKER_TARGET), development)
+	./scripts/check_pip_packages.sh dev.txt
+endif
+# If we expect CI dependencies, then we should check for them
+ifneq ($(INSTALL_CI_DEPS),)
+	./scripts/check_pip_packages.sh ci.txt
+endif
 
 .PHONY: check_files
 check_files: ## check the existence of multiple files
@@ -77,6 +81,26 @@ update_assets:
 	$(PYTHON_COMMAND) manage.py collectstatic --noinput
 
 
+# Build list of dependencies to install
+DEPS = pip prod
+# If we expect CI dependencies, then we should install them
+ifdef INSTALL_CI_DEPS
+DEPS += ci
+endif
+# If we're running a development image, then we should install the development dependencies
+ifeq ($(DOCKER_TARGET), development)
+DEPS += dev
+endif
+
+.PHONY: update_deps
+update_deps: ## Update the dependencies
+# If we're running a non local or production image, remove the existing deps and install clean
+ifeq ($(or $(findstring local, $(DOCKER_TAG)), $(findstring development, $(DOCKER_TARGET))),)
+	rm -rf /deps/**
+endif
+# Install all dependencies in one call
+	$(HOME)/scripts/install_deps.py $(DEPS)
+
 # TOOD: remove this after we migrate addons-frontned to not depend on it.
 .PHONY: setup-ui-tests
 setup-ui-tests:

Makefile-os

@@ -11,6 +11,7 @@ export DOCKER_COMMIT ?=
 export DOCKER_BUILD ?=
 export DOCKER_VERSION ?=
 export DATA_BACKUP_SKIP ?=
+export INSTALL_CI_DEPS ?= True
 override DOCKER_MYSQLD_VOLUME = addons-server_data_mysqld
 
 INITIALIZE_ARGS ?=
@@ -54,7 +55,6 @@ CLEAN_PATHS := \
 	version.json \
 	logs \
 	buildx-bake-metadata.json \
-	deps \
 
 .PHONY: help_redirect
 help_redirect:
@@ -154,8 +154,23 @@ clean_docker: docker_compose_down docker_mysqld_volume_remove docker_clean_image
 docker_compose_up: docker_mysqld_volume_create ## Start the docker containers
 	docker compose up $(DOCKER_COMPOSE_ARGS) $(ARGS)
 
+.PHONY: docker_compose_run
+docker_compose_run: docker_mysqld_volume_create ## Run a command in the docker compose project
+	docker compose run \
+		--rm \
+		--no-deps \
+		$(DOCKER_RUN_ARGS) \
+		web \
+		$(ARGS)
+
+.PHONY: docker_update_deps
+docker_update_deps: ## Update the dependencies in the container based on the docker tag and target
+	mkdir -p deps
+# Install the production dependencies always since we mount the ./deps directory
+	$(MAKE) docker_compose_run ARGS='make update_deps'
+
 .PHONY: up
-up: setup docker_pull_or_build docker_compose_up docker_clean_images docker_clean_volumes ## Create and start docker compose
+up: setup docker_pull_or_build docker_update_deps docker_compose_up docker_clean_images docker_clean_volumes ## Create and start docker compose
 	# Explicitly run initialize via the web container as make can get confused
 	# both routing the command to the web container and
 	# routing the command to the proper target.

docker-compose.yml

@@ -23,6 +23,10 @@ x-env-mapping: &env
     - HOST_UID
     - DEBUG
     - DATA_BACKUP_SKIP
+    - INSTALL_CI_DEPS
+
+x-deps-mount: &deps-mount
+  data_deps:/deps
 
 x-site-static-mount: &site-static-mount
   data_site_static:/data/olympia/site-static
@@ -49,6 +53,7 @@ services:
     # so we just sleep indefinitely instead.
     command: ["sleep", "infinity"]
     volumes:
+      - *deps-mount
       - *site-static-mount
   worker:
     <<: *olympia
@@ -64,6 +69,9 @@ services:
     ]
     volumes:
       - .:/data/olympia
+      # Mount dependencies from the host
+      # make docker_update_deps ensures this volume has correct contents
+      - *deps-mount
 
     extra_hosts:
      - "olympia.test:127.0.0.1"
@@ -81,6 +89,7 @@ services:
       # The start period is 60s
       start_period: 120s
     depends_on:
+      - olympia_volumes
       - mysqld
       - elasticsearch
       - redis
@@ -106,8 +115,6 @@ services:
       - *site-static-mount
       - ./package.json:/deps/package.json
       - ./package-lock.json:/deps/package-lock.json
-    depends_on:
-      - olympia_volumes
 
   nginx:
     image: nginx
@@ -201,6 +208,13 @@ volumes:
   # Volumes for static files that should not be
   # mounted from the host.
   data_site_static:
+  # Volume for dependencies
+  data_deps:
+    driver: local
+    driver_opts:
+      type: none
+      o: bind
+      device: ${PWD}/deps
   data_mysqld:
     # Keep this value in sync with Makefile-os
     # External volumes must be manually created/destroyed