From 771c9987e564b115ee2e09cb3be45204a84fb127 Mon Sep 17 00:00:00 2001 From: Kyle McCormick Date: Tue, 13 Sep 2022 17:14:10 -0400 Subject: [PATCH] feat: overhaul Dockerfile for improved developer workflow WIP --- tutor/templates/build/openedx/Dockerfile | 175 +++++++++++++----- tutor/templates/build/openedx/bin/npm | 14 ++ .../build/openedx/bin/openedx-assets | 93 +++++++++- .../edx-platform-overrides/overrides-test.txt | 1 + .../openedx/dev/mounted-requirements.txt | 0 tutor/templates/dev/docker-compose.yml | 4 +- 6 files changed, 241 insertions(+), 46 deletions(-) create mode 100755 tutor/templates/build/openedx/bin/npm create mode 100644 tutor/templates/build/openedx/dev/edx-platform-overrides/overrides-test.txt create mode 100644 tutor/templates/build/openedx/dev/mounted-requirements.txt diff --git a/tutor/templates/build/openedx/Dockerfile b/tutor/templates/build/openedx/Dockerfile index 4c2dc541ee4..4b2b9358f68 100644 --- a/tutor/templates/build/openedx/Dockerfile +++ b/tutor/templates/build/openedx/Dockerfile @@ -1,27 +1,58 @@ +############################################################################# ###### Minimal image with base system requirements for most stages + FROM docker.io/ubuntu:20.04 as minimal LABEL maintainer="Overhang.io " ENV DEBIAN_FRONTEND=noninteractive +# TODO: Safe/effective to rm var/lib/apt/lists in all these places? RUN apt update && \ - apt install -y build-essential curl git language-pack-en + apt install -y \ + build-essential \ + curl \ + language-pack-en \ + git \ + && \ + rm -rf /var/lib/apt/lists/* ENV LC_ALL en_US.UTF-8 + {{ patch("openedx-dockerfile-minimal") }} -###### Install python with pyenv in /opt/pyenv and create virtualenv in /openedx/venv +############################################################################# +###### Python with pyenv in /opt/pyenv and create virtualenv in /openedx/venv + FROM minimal as python + # https://github.com/pyenv/pyenv/wiki/Common-build-problems#prerequisites RUN apt update && \ - apt install -y libssl-dev zlib1g-dev libbz2-dev \ - libreadline-dev libsqlite3-dev wget curl llvm libncurses5-dev libncursesw5-dev \ - xz-utils tk-dev libffi-dev liblzma-dev python-openssl git + apt install -y \ + curl \ + git \ + libbz2-dev \ + libffi-dev \ + liblzma-dev \ + libncurses5-dev \ + libncursesw5-dev \ + libreadline-dev \ + libsqlite3-dev \ + libssl-dev \ + llvm \ + python-openssl \ + tk-dev \ + wget \ + xz-utils \ + zlib1g-dev \ + && \ + rm -rf /var/lib/apt/lists/* ARG PYTHON_VERSION=3.8.12 ENV PYENV_ROOT /opt/pyenv RUN git clone https://github.com/pyenv/pyenv $PYENV_ROOT --branch v2.2.2 --depth 1 RUN $PYENV_ROOT/bin/pyenv install $PYTHON_VERSION RUN $PYENV_ROOT/versions/$PYTHON_VERSION/bin/python -m venv /openedx/venv -###### Install Dockerize to wait for mysql DB availability +############################################################################# +###### Dockerize stage to wait for mysql DB availability + FROM minimal as dockerize # https://github.com/powerman/dockerize/releases ARG DOCKERIZE_VERSION=v0.16.0 @@ -30,10 +61,13 @@ RUN dockerize_url="https://github.com/powerman/dockerize/releases/download/$DOCK && curl --fail --location --output /usr/local/bin/dockerize $dockerize_url \ && chmod a+x /usr/local/bin/dockerize -###### Checkout edx-platform code +############################################################################# +###### Clone edx-platform in an intermediate stage + FROM minimal as code ARG EDX_PLATFORM_REPOSITORY={{ EDX_PLATFORM_REPOSITORY }} ARG EDX_PLATFORM_VERSION={{ EDX_PLATFORM_VERSION }} +RUN echo BREAK CACHE 1 RUN mkdir -p /openedx/edx-platform && \ git clone $EDX_PLATFORM_REPOSITORY --branch $EDX_PLATFORM_VERSION --depth 1 /openedx/edx-platform WORKDIR /openedx/edx-platform @@ -52,7 +86,9 @@ RUN git config --global user.email "tutor@overhang.io" \ {# Example: RUN curl -fsSL https://github.com/openedx/edx-platform/commit/ | git am #} {{ patch("openedx-dockerfile-post-git-checkout") }} +############################################################################# ###### Download extra locales to /openedx/locale/contrib/locale + FROM minimal as locales ARG OPENEDX_I18N_VERSION={{ OPENEDX_COMMON_VERSION }} RUN cd /tmp \ @@ -62,19 +98,22 @@ RUN cd /tmp \ && mv openedx-i18n-*/edx-platform/locale /openedx/locale/contrib \ && rm -rf openedx-i18n* +############################################################################# ###### Install python requirements in virtualenv + FROM python as python-requirements ENV PATH /openedx/venv/bin:${PATH} ENV VIRTUAL_ENV /openedx/venv/ -RUN apt update && apt install -y software-properties-common libmysqlclient-dev libxmlsec1-dev libgeos-dev - -# Note that this means that we need to reinstall all requirements whenever there is a -# change in edx-platform, which sucks. Yet, we must do it, because edx-platform installs some -# Python projects from within the edx-platform repo itself. This is being fixed upstream. -# TODO: https://github.com/overhangio/2u-tutor-adoption/issues/86 -COPY --from=code /openedx/edx-platform /openedx/edx-platform -WORKDIR /openedx/edx-platform +# TODO: Why are these here instead of an earlier stage? +RUN apt update && \ + apt install -y \ + software-properties-common \ + libmysqlclient-dev \ + libxmlsec1-dev \ + libgeos-dev \ + && \ + rm -rf /var/lib/apt/lists/* # Install the right version of pip/setuptools # https://pypi.org/project/setuptools/ @@ -82,11 +121,7 @@ WORKDIR /openedx/edx-platform # https://pypi.org/project/wheel/ RUN pip install setuptools==62.1.0 pip==22.0.4 wheel==0.37.1 -# Install base requirements -RUN pip install -r ./requirements/edx/base.txt -RUN pip install -e . - -# Install django-redis for using redis as a django cache +# Iedxnstall django-redis for using redis as a django cache # https://pypi.org/project/django-redis/ RUN pip install django-redis==5.2.0 @@ -94,6 +129,11 @@ RUN pip install django-redis==5.2.0 # https://pypi.org/project/uWSGI/ RUN pip install uwsgi==2.0.20 +# Install base requirements +COPY --from=code /openedx/edx-platform/requirements /openedx/edx-platform/requirements +WORKDIR /openedx/edx-platform +RUN pip install -r ./requirements/edx/base.txt + {{ patch("openedx-dockerfile-post-python-requirements") }} # Install private requirements: this is useful for installing custom xblocks. @@ -105,7 +145,9 @@ RUN cd /openedx/requirements/ \ {% for extra_requirements in OPENEDX_EXTRA_PIP_REQUIREMENTS %}RUN pip install '{{ extra_requirements }}' {% endfor %} +############################################################################# ###### Install nodejs with nodeenv in /openedx/nodeenv + FROM python as nodejs-requirements ENV PATH /openedx/nodeenv/bin:/openedx/venv/bin:${PATH} @@ -120,36 +162,66 @@ COPY --from=code /openedx/edx-platform/package-lock.json /openedx/edx-platform/p WORKDIR /openedx/edx-platform RUN npm install --verbose --registry=$NPM_REGISTRY +############################################################################# ###### Production image with system and python requirements + FROM minimal as production # Install system requirements RUN apt update && \ - apt install -y gettext gfortran graphviz graphviz-dev libffi-dev libfreetype6-dev libgeos-dev libjpeg8-dev liblapack-dev libmysqlclient-dev libpng-dev libsqlite3-dev libxmlsec1-dev lynx ntp pkg-config rdfind && \ + apt install -y \ + gettext \ + gfortran \ + graphviz \ + graphviz-dev \ + libffi-dev \ + libfreetype6-dev \ + libgeos-dev \ + libjpeg8-dev \ + liblapack-dev \ + libmysqlclient-dev \ + libpng-dev \ + libsqlite3-dev \ + libxmlsec1-dev \ + lynx \ + ntp \ + pkg-config \ + rdfind \ + && \ rm -rf /var/lib/apt/lists/* -# From then on, run as unprivileged "app" user +# From here on, run as unprivileged "app" user # Note that this must always be different from root (APP_USER_ID=0) ARG APP_USER_ID=1000 RUN if [ "$APP_USER_ID" = 0 ]; then echo "app user may not be root" && false; fi RUN useradd --home-dir /openedx --create-home --shell /bin/bash --uid ${APP_USER_ID} app USER ${APP_USER_ID} -COPY --from=dockerize /usr/local/bin/dockerize /usr/local/bin/dockerize +# Copy in everything we need from the intermediate build stages +COPY --from=dockerize /usr/local/bin/dockerize /usr/local/bin/dockerize COPY --chown=app:app --from=code /openedx/edx-platform /openedx/edx-platform COPY --chown=app:app --from=locales /openedx/locale /openedx/locale COPY --chown=app:app --from=python /opt/pyenv /opt/pyenv COPY --chown=app:app --from=python-requirements /openedx/venv /openedx/venv COPY --chown=app:app --from=python-requirements /openedx/requirements /openedx/requirements COPY --chown=app:app --from=nodejs-requirements /openedx/nodeenv /openedx/nodeenv -COPY --chown=app:app --from=nodejs-requirements /openedx/edx-platform/node_modules /openedx/edx-platform/node_modules +COPY --chown=app:app --from=nodejs-requirements /openedx/edx-platform/node_modules /openedx/node_modules +RUN mv /openedx/node_modules/.bin /openedx/node_modules/bin -ENV PATH /openedx/venv/bin:./node_modules/.bin:/openedx/nodeenv/bin:${PATH} +# Enable venv & nodeenv +ENV PATH /openedx/venv/bin:/openedx/node_modules/bin:/openedx/nodeenv/bin:${PATH} ENV VIRTUAL_ENV /openedx/venv/ WORKDIR /openedx/edx-platform -# Re-install local requirements, otherwise egg-info folders are missing -RUN pip install -r requirements/edx/local.in +# Copy scripts and put them on the path. +COPY --chown=app:app ./bin /openedx/bin +RUN chmod a+x /openedx/bin/* +ENV PATH /openedx/bin:${PATH} + +# Generate Open-edX.egg_info by installing edx-platform itself. +# Ensures entrypoints & console scripts from setup.py are registered. +# TODO: Is it worth fixing up setup.py so that we can just run `pip install .` ? +RUN pip install -e . # Create folder that will store lms/cms.env.yml files, as well as # the tutor-specific settings files. @@ -173,11 +245,6 @@ RUN cd /openedx/locale/user && \ RUN ./manage.py lms --settings=tutor.i18n compilejsi18n RUN ./manage.py cms --settings=tutor.i18n compilejsi18n -# Copy scripts -COPY --chown=app:app ./bin /openedx/bin -RUN chmod a+x /openedx/bin/* -ENV PATH /openedx/bin:${PATH} - {{ patch("openedx-dockerfile-pre-assets") }} # Collect production assets. By default, only assets from the default theme @@ -189,13 +256,19 @@ ENV PATH /openedx/bin:${PATH} # /openedx/staticfiles. ENV NO_PYTHON_UNINSTALL 1 ENV NO_PREREQ_INSTALL 1 + +# TODO: This is necessary for _compile_sass currently but should be removed +RUN ln -s /openedx/node_modules +RUN cd node_modules && ln -s bin .bin + # We need to rely on a separate openedx-assets command to accelerate asset processing. # For instance, we don't want to run all steps of asset collection every time the theme # is modified. +# TODO: Can we remove the pavelib dependency? RUN openedx-assets xmodule \ - && openedx-assets npm \ - && openedx-assets webpack --env=prod \ - && openedx-assets common + && openedx-assets npm \ + && openedx-assets webpack --env=prod \ + && openedx-assets common COPY --chown=app:app ./themes/ /openedx/themes/ RUN openedx-assets themes \ && openedx-assets collect --settings=tutor.assets \ @@ -213,37 +286,55 @@ ENV DJANGO_SETTINGS_MODULE lms.envs.tutor.production EXPOSE 8000 +############################################################################# ###### Intermediate image with dev/test dependencies + FROM production as development # Install useful system requirements (as root) USER root RUN apt update && \ - apt install -y vim iputils-ping dnsutils telnet \ - && rm -rf /var/lib/apt/lists/* + apt install -y \ + dnsutils \ + iputils-ping \ + telnet \ + vim \ + && \ + rm -rf /var/lib/apt/lists/* USER app -# Install dev python requirements -RUN pip install -r requirements/edx/development.txt -RUN pip install ipdb==0.13.4 ipython==7.27.0 - # Add ipdb as default PYTHONBREAKPOINT +RUN pip install ipdb==0.13.4 ipython==7.27.0 ENV PYTHONBREAKPOINT=ipdb.set_trace +# Override any edx-platform files, as supplied by build context. +# Primarily, this allows Tutor to override /openedx/edx-platform/requirements +# when edx-platform is mounted in order to force a requirements re-install +# and static asset re-compilation below. +COPY ./dev/edx-platform-overrides /openedx/edx-platform + +# Install dev requirements +RUN pip install -r requirements/edx/development.txt +RUN npm install --dev + # Recompile static assets: in development mode all static assets are stored in edx-platform, # and the location of these files is stored in webpack-stats.json. If we don't recompile # static assets, then production assets will be served instead. +# TODO: OK to use scripts/update-assets-dev.sh? RUN rm -r /openedx/staticfiles && \ mkdir /openedx/staticfiles && \ openedx-assets webpack --env=dev +# scripts/update-assets-dev.sh {{ patch("openedx-dev-dockerfile-post-python-requirements") }} # Default django settings ENV DJANGO_SETTINGS_MODULE lms.envs.tutor.development -CMD ./manage.py $SERVICE_VARIANT runserver 0.0.0.0:8000 +COPY ./dev/mounted-requirements.txt /openedx/mounted-requirements.txt +CMD pip install -r /openedx/mounted-requirements.txt && ./manage.py $SERVICE_VARIANT runserver 0.0.0.0:8000 +############################################################################# ###### Final image with production cmd FROM production as final diff --git a/tutor/templates/build/openedx/bin/npm b/tutor/templates/build/openedx/bin/npm new file mode 100755 index 00000000000..bb2ceb94480 --- /dev/null +++ b/tutor/templates/build/openedx/bin/npm @@ -0,0 +1,14 @@ +#!/bin/sh +# +# Wrapper around 'npm' for openedx image. +# Ensures that npm always operates in the 'global' /openedx/node_modules directory, +# not in the 'local' /openedx/edx-platform/node_modules directory. +# +# Rationale: We want node_modules to exist outside of the edx-platform repository. +# That way, when a developer mounts a fork of edx-platform, they don't need to +# re-run `npm install` themselves, because the pre-built node_modules folder will +# still be available on the image. + +set -ue # Fail loudly. +set -x # Print next command. +/openedx/nodeenv/bin/npm --global --prefix=/openedx/node_modules "$@" diff --git a/tutor/templates/build/openedx/bin/openedx-assets b/tutor/templates/build/openedx/bin/openedx-assets index 1b89434d672..b2b5d0969bc 100755 --- a/tutor/templates/build/openedx/bin/openedx-assets +++ b/tutor/templates/build/openedx/bin/openedx-assets @@ -2,6 +2,7 @@ from __future__ import print_function import argparse import os +import shlex import subprocess import sys import traceback @@ -98,6 +99,8 @@ def run_build(args): def run_xmodule(_args): + print(f"{sys.argv[0]}: Collecting xmodule assets") + # Collecting xmodule assets is incompatible with setting the django path, because # of an unfortunate call to settings.configure() django_settings_module = os.environ.get("DJANGO_SETTINGS_MODULE") @@ -114,14 +117,93 @@ def run_xmodule(_args): def run_npm(_args): - assets.process_npm_assets() + """ + Post-process npm assets. + + Reimplementation of edx-platform's pavelib/assets.py:process_npm_assets() + """ + print(f"{sys.argv[0]}: Post-processing npm assets") + + # Create JS and CSS vendor directories. + _sh( + [ + "mkdir", + "-p", + "common/static/common/js/vendor", + "common/static/common/css/vendor", + ] + ) + + # Copy studio-frontend CSS and JS into vendor directory. + copy_css = """\ +find /openedx/node_modules/@edx/studio-frontend/dist \ +-type f \( -name \*.css -o -name \*.css.map \) | \ +xargs cp --target-directory=common/static/common/css/vendor\ +""" + copy_js = """\ +find /openedx/node_modules/@edx/studio-frontend/dist \ +-type f \! -name \*.css \! -name \*.css.map | \ +xargs cp --target-directory=common/static/common/js/vendor\ +""" + _sh(["bash", "-c", copy_css]) + _sh(["bash", "-c", copy_js]) +# css_find_query = "-type f '(' -name '*.css' -o -name '*.css.map' ')'".split(" ") +# js_find_query = "-type f '!' -name '*.css' '!' -name '*.css.map'".split(" ") +# _sh( +# [ +# "find", +# "/openedx/node_modules/@edx/studio-frontend/dist", +# *css_find_query, +# "|", +# "xargs", +# "cp", +# "--target-directory=common/static/common/css/vender", +# ] +# ) +# _sh( +# [ +# "find", +# "/openedx/node_modules/@edx/studio-frontend/dist", +# *js_find_query, +# "|", +# "xargs", +# "cp", +# "--target-directory=common/static/common/js/vender", +# ] +# ) + + # Copy certain node_modules into vendor directory. + _sh( + [ + "cp", + "-f", + "--target-directory=common/static/common/js/vendor", + "/openedx/node_modules/backbone.paginator/lib/backbone.paginator.js", + "/openedx/node_modules/backbone/backbone.js", + "/openedx/node_modules/bootstrap/dist/js/bootstrap.bundle.js", + "/openedx/node_modules/hls.js/dist/hls.js", + "/openedx/node_modules/jquery-migrate/dist/jquery-migrate.js", + "/openedx/node_modules/jquery.scrollto/jquery.scrollTo.js", + "/openedx/node_modules/jquery/dist/jquery.js", + "/openedx/node_modules/moment-timezone/builds/moment-timezone-with-data.js", + "/openedx/node_modules/moment/min/moment-with-locales.js", + "/openedx/node_modules/picturefill/dist/picturefill.js", + "/openedx/node_modules/requirejs/require.js", + "/openedx/node_modules/underscore.string/dist/underscore.string.js", + "/openedx/node_modules/underscore/underscore.js", + "/openedx/node_modules/which-country/index.js", + "/openedx/node_modules/sinon/pkg/sinon.js", + "/openedx/node_modules/squirejs/src/Squire.js", + ] + ) def run_webpack(args): + print(f"{sys.argv[0]}: Executing webpack") os.environ["STATIC_ROOT_LMS"] = args.static_root os.environ["STATIC_ROOT_CMS"] = os.path.join(args.static_root, "studio") os.environ["NODE_ENV"] = {"prod": "production", "dev": "development"}[args.env] - subprocess.check_call( + _sh( [ "webpack", "--progress", @@ -131,12 +213,14 @@ def run_webpack(args): def run_common(args): + print(f"{sys.argv[0]}: Compiling sass assets from common theme") for system in args.systems: print("Compiling {} sass assets from common theme...".format(system)) assets._compile_sass(system, None, False, False, []) def run_themes(args): + print(f"{sys.argv[0]}: Compiling sass assets for custom themes") for theme_dir in args.theme_dirs: local_themes = ( list_subdirectories(theme_dir) if "all" in args.themes else args.themes @@ -154,6 +238,7 @@ def run_themes(args): def run_collect(args): + print(f"{sys.argv[0]}: Collecting assets") assets.collect_assets(args.systems, args.settings) @@ -214,5 +299,9 @@ class ThemeWatcher(assets.SassWatcher): traceback.print_exc() +def _sh(args): + print(f"+{shlex.join(args)}") + return subprocess.check_call(args) + if __name__ == "__main__": main() diff --git a/tutor/templates/build/openedx/dev/edx-platform-overrides/overrides-test.txt b/tutor/templates/build/openedx/dev/edx-platform-overrides/overrides-test.txt new file mode 100644 index 00000000000..3d9880a12c8 --- /dev/null +++ b/tutor/templates/build/openedx/dev/edx-platform-overrides/overrides-test.txt @@ -0,0 +1 @@ +edx-platform-overrides test file diff --git a/tutor/templates/build/openedx/dev/mounted-requirements.txt b/tutor/templates/build/openedx/dev/mounted-requirements.txt new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tutor/templates/dev/docker-compose.yml b/tutor/templates/dev/docker-compose.yml index 58b27aa8329..7691460b497 100644 --- a/tutor/templates/dev/docker-compose.yml +++ b/tutor/templates/dev/docker-compose.yml @@ -30,7 +30,7 @@ services: lms: <<: *openedx-service - command: ./manage.py lms runserver 0.0.0.0:8000 + command: pip install /openedx/mounted-requirements.txt && ./manage.py lms runserver 0.0.0.0:8000 environment: DJANGO_SETTINGS_MODULE: lms.envs.tutor.development ports: @@ -42,7 +42,7 @@ services: cms: <<: *openedx-service - command: ./manage.py cms runserver 0.0.0.0:8000 + command: pip install /openedx/mounted-requirements.txt && ./manage.py cms runserver 0.0.0.0:8000 environment: DJANGO_SETTINGS_MODULE: cms.envs.tutor.development ports: