Skip to content

Commit

Permalink
Add deps mount to mount dependencies from host
Browse files Browse the repository at this point in the history
  • Loading branch information
KevinMind committed Dec 11, 2024
1 parent 28390a8 commit ad41110
Show file tree
Hide file tree
Showing 5 changed files with 131 additions and 21 deletions.
14 changes: 5 additions & 9 deletions Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -57,10 +57,6 @@ chown -R olympia:olympia /deps
ln -s /deps/bin/uwsgi /usr/bin/uwsgi
ln -s /usr/bin/uwsgi /usr/sbin/uwsgi

# link to the package*.json at ${HOME} so npm can install in /deps
ln -s ${HOME}/package.json /deps/package.json
ln -s ${HOME}/package-lock.json /deps/package-lock.json

# Create the storage directory and the test file to verify nginx routing
mkdir -p ${HOME}/storage
chown -R olympia:olympia ${HOME}/storage
Expand All @@ -86,13 +82,12 @@ ENV NPM_ARGS="--prefix ${NPM_CONFIG_PREFIX} --cache ${NPM_CACHE_DIR} --loglevel
# All we need in "base" is pip to be installed
#this let's other layers install packages using the correct version.
RUN \
--mount=type=bind,source=scripts/install_deps.py,target=${HOME}/scripts/install_deps.py \
# Files required to install pip dependencies
--mount=type=bind,source=./requirements/pip.txt,target=${HOME}/requirements/pip.txt \
--mount=type=cache,target=${PIP_CACHE_DIR},uid=${OLYMPIA_UID},gid=${OLYMPIA_UID} \
<<EOF
# Work arounds "Multiple .dist-info directories" issue.
rm -rf /deps/build/*
${PIP_COMMAND} install --progress-bar=off --no-deps --exists-action=w -r requirements/pip.txt
${HOME}/scripts/install_deps.py pip
EOF

# Expose the DOCKER_TARGET variable to all subsequent stages
Expand All @@ -106,6 +101,7 @@ ENV DOCKER_TARGET=${DOCKER_TARGET}
FROM base AS pip_production

RUN \
--mount=type=bind,source=scripts/install_deps.py,target=${HOME}/scripts/install_deps.py \
# Files required to install pip dependencies
--mount=type=bind,source=./requirements/prod.txt,target=${HOME}/requirements/prod.txt \
# Files required to install npm dependencies
Expand All @@ -115,13 +111,13 @@ RUN \
--mount=type=cache,target=${PIP_CACHE_DIR},uid=${OLYMPIA_UID},gid=${OLYMPIA_UID} \
--mount=type=cache,target=${NPM_CACHE_DIR},uid=${OLYMPIA_UID},gid=${OLYMPIA_UID} \
<<EOF
${PIP_COMMAND} install --progress-bar=off --no-deps --exists-action=w -r requirements/prod.txt
npm ci ${NPM_ARGS} --include=prod
${HOME}/scripts/install_deps.py prod
EOF

FROM pip_production AS pip_development

RUN \
--mount=type=bind,source=scripts/install_deps.py,target=${HOME}/scripts/install_deps.py \
# Files required to install pip dependencies
--mount=type=bind,source=./requirements/prod.txt,target=${HOME}/requirements/prod.txt \
--mount=type=bind,source=./requirements/dev.txt,target=${HOME}/requirements/dev.txt \
Expand Down
26 changes: 20 additions & 6 deletions Makefile-docker
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,13 @@ REQUIRED_FILES := \
/deps/package-lock.json \
/addons-server-docker-container \

# Build list of dependencies to install
DEPS = pip prod
# If we're running a development image, then we should install the development dependencies
ifeq ($(DOCKER_TARGET), development)
DEPS += dev
endif

.PHONY: help_redirect
help_redirect:
@$(MAKE) help --no-print-directory
Expand All @@ -28,12 +35,7 @@ check_debian_packages: ## check the existence of multiple debian packages

.PHONY: check_pip_packages
check_pip_packages: ## check the existence of multiple python packages
@ ./scripts/check_pip_packages.sh prod.txt
# "production" corresponds to the "propduction" DOCKER_TARGET defined in the Dockerfile
# When the target is "production" it means we cannot expect dev.txt dependencies to be installed.
@if [ "$(DOCKER_TARGET)" != "production" ]; then \
./scripts/check_pip_packages.sh dev.txt; \
fi
$(foreach dep, $(DEPS), ./scripts/check_pip_packages.sh $(dep).txt)

.PHONY: check_files
check_files: ## check the existence of multiple files
Expand Down Expand Up @@ -77,6 +79,18 @@ update_assets:
$(PYTHON_COMMAND) manage.py collectstatic --noinput


.PHONY: update_deps
update_deps: ## Update the dependencies
# If we're running a non local or production image, remove the existing deps and install clean
ifneq ($(and $(findstring local, $(DOCKER_TAG)), $(findstring development, $(DOCKER_TARGET))),)
@echo "Updating existing deps"
else
@echo "Removing existing deps"
rm -rf /deps/**
endif
# Install all dependencies in one call
$(HOME)/scripts/install_deps.py $(DEPS)

# TOOD: remove this after we migrate addons-frontned to not depend on it.
.PHONY: setup-ui-tests
setup-ui-tests:
Expand Down
18 changes: 16 additions & 2 deletions Makefile-os
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,6 @@ CLEAN_PATHS := \
version.json \
logs \
buildx-bake-metadata.json \
deps \

.PHONY: help_redirect
help_redirect:
Expand Down Expand Up @@ -154,8 +153,23 @@ clean_docker: docker_compose_down docker_mysqld_volume_remove docker_clean_image
docker_compose_up: docker_mysqld_volume_create ## Start the docker containers
docker compose up $(DOCKER_COMPOSE_ARGS) $(ARGS)

.PHONY: docker_compose_run
docker_compose_run: docker_mysqld_volume_create ## Run a command in the docker compose project
docker compose run \
--rm \
--no-deps \
$(DOCKER_RUN_ARGS) \
web \
$(ARGS)

.PHONY: docker_update_deps
docker_update_deps: ## Update the dependencies in the container based on the docker tag and target
mkdir -p deps
# Install the production dependencies always since we mount the ./deps directory
$(MAKE) docker_compose_run ARGS='make update_deps'

.PHONY: up
up: setup docker_pull_or_build docker_compose_up docker_clean_images docker_clean_volumes ## Create and start docker compose
up: setup docker_pull_or_build docker_update_deps docker_compose_up docker_clean_images docker_clean_volumes ## Create and start docker compose
# Explicitly run initialize via the web container as make can get confused
# both routing the command to the web container and
# routing the command to the proper target.
Expand Down
19 changes: 15 additions & 4 deletions docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,9 @@ x-env-mapping: &env
- DEBUG
- DATA_BACKUP_SKIP

x-deps-mount: &deps-mount
data_deps:/deps

x-site-static-mount: &site-static-mount
data_site_static:/data/olympia/site-static

Expand All @@ -49,6 +52,7 @@ services:
# so we just sleep indefinitely instead.
command: ["sleep", "infinity"]
volumes:
- *deps-mount
- *site-static-mount
worker:
<<: *olympia
Expand All @@ -64,6 +68,9 @@ services:
]
volumes:
- .:/data/olympia
# Mount dependencies from the host
# make docker_update_deps ensures this volume has correct contents
- *deps-mount

extra_hosts:
- "olympia.test:127.0.0.1"
Expand All @@ -81,6 +88,7 @@ services:
# The start period is 60s
start_period: 120s
depends_on:
- olympia_volumes
- mysqld
- elasticsearch
- redis
Expand All @@ -104,10 +112,6 @@ services:
# and would otherwiser be deleted by mounting the cwd volume above
- /data/olympia/static-build
- *site-static-mount
- ./package.json:/deps/package.json
- ./package-lock.json:/deps/package-lock.json
depends_on:
- olympia_volumes

nginx:
image: nginx
Expand Down Expand Up @@ -201,6 +205,13 @@ volumes:
# Volumes for static files that should not be
# mounted from the host.
data_site_static:
# Volume for dependencies
data_deps:
driver: local
driver_opts:
type: none
o: bind
device: ${PWD}/deps
data_mysqld:
# Keep this value in sync with Makefile-os
# External volumes must be manually created/destroyed
Expand Down
75 changes: 75 additions & 0 deletions scripts/install_deps.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
#!/usr/bin/env python3

import os
import shutil
import subprocess
import sys


# Constants
ALLOWED_NPM_TARGETS = set(['prod', 'dev'])


def copy_package_json():
"""Copy package.json files to deps directory if they exist."""
try:
shutil.copy('/data/olympia/package.json', '/deps')
shutil.copy('/data/olympia/package-lock.json', '/deps')
except (IOError, OSError):
pass # Ignore if files don't exist or can't be copied


def main():
# Get targets from command line arguments
targets = sys.argv[1:]
if not targets:
print('No targets specified')
sys.exit(1)

# Copy package.json files
copy_package_json()

# Prepare the includes lists
pip_includes = []
npm_includes = []

# PIP_COMMAND is set by the Dockerfile
pip_command = os.environ.get('PIP_COMMAND', 'pip')
pip_args = pip_command.split() + [
'install',
'--progress-bar=off',
'--no-deps',
'--exists-action=w',
]

# NPM_ARGS is set by the Dockerfile
npm_args_env = os.environ.get('NPM_ARGS', '')
npm_args = [
'npm',
'install',
'--no-save',
'--no-audit',
'--no-fund',
] + npm_args_env.split()

# Add the relevant targets to the includes lists
for target in targets:
pip_includes.append(target)
pip_args.extend(['-r', f'requirements/{target}.txt'])
if target in ALLOWED_NPM_TARGETS:
npm_includes.append(target)
npm_args.extend(['--include', target])

if pip_includes:
# Install pip dependencies
print(f"Installing pip dependencies: {', '.join(pip_includes)} \n")
subprocess.run(pip_args, check=True)

if npm_includes:
# Install npm dependencies
print(f"Installing npm dependencies: {', '.join(npm_includes)} \n")
subprocess.run(npm_args, check=True)


if __name__ == '__main__':
main()

0 comments on commit ad41110

Please sign in to comment.