From 69803944dc8771875b143379c2826dfe61af75da Mon Sep 17 00:00:00 2001 From: Kunal Mehta Date: Wed, 31 Aug 2022 15:06:44 -0400 Subject: [PATCH] Build and install redwood wheel in all developer tooling We need to compile Rust code before we can launch the development environment and run tests. Add a build_redwood step to `securedrop/bin/dev-deps` that uses maturin to build a wheel and install it into the securedrop-app-code virtualenv. maturin is only needed at build time, so it is pinned in a new `build-requirements.txt` Fixes #6814. Fixes #6816. --- Makefile | 4 +++ securedrop/bin/dev-deps | 12 +++++++++ securedrop/bin/generate-docs-screenshots | 1 + securedrop/bin/run | 1 + securedrop/bin/run-mypy | 3 +++ securedrop/bin/run-test | 1 + securedrop/bin/translation-test | 1 + .../dockerfiles/focal/python3/SlimDockerfile | 22 ++++++++++++++-- .../python3/build-requirements.in | 1 + .../python3/build-requirements.txt | 25 +++++++++++++++++++ 10 files changed, 69 insertions(+), 2 deletions(-) create mode 100644 securedrop/requirements/python3/build-requirements.in create mode 100644 securedrop/requirements/python3/build-requirements.txt diff --git a/Makefile b/Makefile index cd7ebc2512e..59d6f9dcf3d 100644 --- a/Makefile +++ b/Makefile @@ -30,6 +30,10 @@ update-admin-pip-requirements: ## Update admin requirements. update-python3-requirements: ## Update Python 3 requirements with pip-compile. @echo "███ Updating Python 3 requirements files..." @SLIM_BUILD=1 $(DEVSHELL) pip-compile --generate-hashes \ + --allow-unsafe \ + --output-file requirements/python3/build-requirements.txt \ + requirements/python3/build-requirements.in + @$(DEVSHELL) pip-compile --generate-hashes \ --allow-unsafe \ --output-file requirements/python3/develop-requirements.txt \ ../admin/requirements-ansible.in \ diff --git a/securedrop/bin/dev-deps b/securedrop/bin/dev-deps index a4262b27447..468a44eba29 100755 --- a/securedrop/bin/dev-deps +++ b/securedrop/bin/dev-deps @@ -156,3 +156,15 @@ function reset_demo() { ./loaddata.py fi } + +function build_redwood() { + # Create a debug build of redwood and install it in the app-code virtualenv + # TODO: add live reload support + # We tell cargo to put its registry/crate cache and build cache in the target/ folder so + # it persists instead of being recreated from scratch each time the container starts. + # n.b. we can't re-use the host's caches because of permission differences + PATH="$PATH:/opt/cargo/bin/" \ + CARGO_TARGET_DIR="${REPOROOT}/target/dev" CARGO_HOME="${REPOROOT}/target/cargo-dev" \ + /opt/venvs/securedrop-build/bin/maturin build -m "${REPOROOT}/redwood/Cargo.toml" + /opt/venvs/securedrop-app-code/bin/pip install "${REPOROOT}"/target/dev/wheels/redwood-*.whl +} diff --git a/securedrop/bin/generate-docs-screenshots b/securedrop/bin/generate-docs-screenshots index 3b34a040a2d..93f8aff29dd 100755 --- a/securedrop/bin/generate-docs-screenshots +++ b/securedrop/bin/generate-docs-screenshots @@ -10,6 +10,7 @@ run_tor & run_redis & run_x11vnc & urandom +build_redwood maybe_create_config_py ./i18n_tool.py translate-messages --compile diff --git a/securedrop/bin/run b/securedrop/bin/run index fc8c57150c8..cd32d4fe4f4 100755 --- a/securedrop/bin/run +++ b/securedrop/bin/run @@ -14,6 +14,7 @@ source "${BASH_SOURCE%/*}/dev-deps" run_redis & urandom +build_redwood maybe_create_config_py reset_demo maybe_use_tor diff --git a/securedrop/bin/run-mypy b/securedrop/bin/run-mypy index c95b41c149f..a625e13a7d1 100755 --- a/securedrop/bin/run-mypy +++ b/securedrop/bin/run-mypy @@ -7,6 +7,9 @@ REPOROOT=$(git rev-parse --show-toplevel) cd "${REPOROOT}" if [ "$(command -v mypy)" ]; then + source "${BASH_SOURCE%/*}/dev-deps" + build_redwood + mypy ./securedrop ./admin --namespace-packages --explicit-package-bases "$@" elif [ -d "/opt/venvs/securedrop-app-code/" ]; then # Inside the dev container, but no mypy diff --git a/securedrop/bin/run-test b/securedrop/bin/run-test index 0ea72fe53f8..ff11dc33f54 100755 --- a/securedrop/bin/run-test +++ b/securedrop/bin/run-test @@ -24,6 +24,7 @@ run_redis & setup_vncauth run_x11vnc & urandom +build_redwood maybe_create_config_py if [ -n "${CIRCLE_BRANCH:-}" ] ; then diff --git a/securedrop/bin/translation-test b/securedrop/bin/translation-test index 00dc5a4e6ef..5cc6303bdc7 100755 --- a/securedrop/bin/translation-test +++ b/securedrop/bin/translation-test @@ -11,6 +11,7 @@ run_redis & setup_vncauth run_x11vnc & urandom +build_redwood maybe_create_config_py ./i18n_tool.py translate-messages --compile diff --git a/securedrop/dockerfiles/focal/python3/SlimDockerfile b/securedrop/dockerfiles/focal/python3/SlimDockerfile index f5d0e7802b0..3dec3a5bd62 100644 --- a/securedrop/dockerfiles/focal/python3/SlimDockerfile +++ b/securedrop/dockerfiles/focal/python3/SlimDockerfile @@ -9,16 +9,34 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get install apache2-dev coreutils vim \ python3-pip python3-all python3-venv virtualenv python3-dev libssl-dev \ gnupg2 redis-server git curl wget \ - enchant libffi-dev sqlite3 gettext sudo tor basez + enchant libffi-dev sqlite3 gettext sudo tor basez pkg-config + +# Install Rust +ENV RUST_VERSION 1.69.0 +ENV RUSTUP_VERSION 1.24.3 +ENV RUSTUP_INIT_SHA256 3dc5ef50861ee18657f9db2eeb7392f9c2a6c95c90ab41e45ab4ca71476b4338 +ENV RUSTUP_HOME /opt/rustup +ENV CARGO_HOME /opt/cargo + +RUN TMPDIR=`mktemp -d` && cd ${TMPDIR} \ + && curl --proto '=https' --tlsv1.2 -OO -sSf https://static.rust-lang.org/rustup/archive/${RUSTUP_VERSION}/x86_64-unknown-linux-gnu/rustup-init \ + && echo "${RUSTUP_INIT_SHA256} *rustup-init" | sha256sum -c - \ + && chmod +x rustup-init \ + && ./rustup-init --default-toolchain=${RUST_VERSION} --profile minimal -y \ + && cd && rm -rf ${TMPDIR} +# TODO: Remove after we switch to 1.70.0 when it's enabled by default +ENV CARGO_REGISTRIES_CRATES_IO_PROTOCOL sparse COPY requirements requirements +RUN python3 -m venv /opt/venvs/securedrop-build && \ + /opt/venvs/securedrop-build/bin/pip3 install --no-deps --require-hashes -r requirements/python3/build-requirements.txt RUN python3 -m venv /opt/venvs/securedrop-app-code && \ /opt/venvs/securedrop-app-code/bin/pip3 install --no-deps --require-hashes -r requirements/python3/bootstrap-requirements.txt && \ /opt/venvs/securedrop-app-code/bin/pip3 install --no-deps --require-hashes -r requirements/python3/test-requirements.txt && \ /opt/venvs/securedrop-app-code/bin/pip3 install --no-deps --require-hashes -r requirements/python3/requirements.txt RUN if test $USER_NAME != root ; then useradd --no-create-home --home-dir /tmp --uid $USER_ID $USER_NAME && echo "$USER_NAME ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers ; fi && \ - chown -R $USER_NAME.$USER_NAME /opt/venvs/securedrop-app-code/ + chown -R $USER_NAME.$USER_NAME /opt/venvs/ STOPSIGNAL SIGKILL diff --git a/securedrop/requirements/python3/build-requirements.in b/securedrop/requirements/python3/build-requirements.in new file mode 100644 index 00000000000..62fc9ff5f42 --- /dev/null +++ b/securedrop/requirements/python3/build-requirements.in @@ -0,0 +1 @@ +maturin>=1.0,<2.0 diff --git a/securedrop/requirements/python3/build-requirements.txt b/securedrop/requirements/python3/build-requirements.txt new file mode 100644 index 00000000000..8d31e05cf1a --- /dev/null +++ b/securedrop/requirements/python3/build-requirements.txt @@ -0,0 +1,25 @@ +# +# This file is autogenerated by pip-compile +# To update, run: +# +# pip-compile --allow-unsafe --generate-hashes --output-file=requirements/python3/build-requirements.txt requirements/python3/build-requirements.in +# +maturin==1.0.1 \ + --hash=sha256:04c0279dd0d6ccd317018bd1a43f52cbda715822537ae1a68015c9171f18b2fd \ + --hash=sha256:10097e2602330c0b9db16d7dfd002476f5e5cf99df58ba2f3abc6de64a69e9a6 \ + --hash=sha256:2907b345186a83db4bbe5571830509b3031784d08958b32d2ffa7857bd473725 \ + --hash=sha256:6b020b9abbd1e9fef468c171216dc4be053834b5bf638075264ee090a993b0b0 \ + --hash=sha256:6d9b4ff7c2d501e91886b859296f5c0478fc08bc7d537a72f98a69d51ff4f519 \ + --hash=sha256:71fdb2dbbd5bcc60bd91ddcbe34dba9f04cc53c2add089a95a79d0d8fc8337b8 \ + --hash=sha256:787bb56c80eda482ece2dd4788d479dbd0e74d981b2e2c538228365c19290fb7 \ + --hash=sha256:8d88d1595d7514c27df96d5f4fe3dc5f24288528a746439403f27c3b448fca16 \ + --hash=sha256:9ecebccb111c9c870fb2f5eee17518fe106f676227bb16f204a51e7a162aceec \ + --hash=sha256:b39f9a42b3c8242e3f3ab990bd03ba989c6c07e4de9e21fcf877a2418119d445 \ + --hash=sha256:c0b1efa47f8b7d15bc5945159764ce57316f9d1bfb7c8caa07cebdd41318359b \ + --hash=sha256:d271b24febbfc020561984b1acdfc39b132df21f4e42d7af0fe274ea738c8000 \ + --hash=sha256:d392ec0578d9e6f03914837cef7bbb264d5708807e0b48176b6ff0b50083ba7c + # via -r requirements/python3/build-requirements.in +tomli==2.0.1 \ + --hash=sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc \ + --hash=sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f + # via maturin