Build and install redwood wheel in all developer tooling

We need to compile Rust code before we can launch the development environment and run tests. Add a build_redwood step to `securedrop/bin/dev-deps` that uses maturin to build a wheel and install it into the securedrop-app-code virtualenv. maturin is only needed at build time, so it is pinned in a new `build-requirements.txt` Fixes #6814. Fixes #6816.
freedomofpress · Jun 6, 2023 · 6980394 · 6980394
1 parent dc9413a
commit 6980394
Show file tree

Hide file tree

Showing 10 changed files with 69 additions and 2 deletions.
diff --git a/Makefile b/Makefile
@@ -30,6 +30,10 @@ update-admin-pip-requirements:  ## Update admin requirements.
 update-python3-requirements:  ## Update Python 3 requirements with pip-compile.
 	@echo "███ Updating Python 3 requirements files..."
 	@SLIM_BUILD=1 $(DEVSHELL) pip-compile --generate-hashes \
+		--allow-unsafe \
+		--output-file requirements/python3/build-requirements.txt \
+		requirements/python3/build-requirements.in
+	@$(DEVSHELL) pip-compile --generate-hashes \
 		--allow-unsafe \
 		--output-file requirements/python3/develop-requirements.txt \
 		../admin/requirements-ansible.in \

diff --git a/securedrop/bin/dev-deps b/securedrop/bin/dev-deps
@@ -156,3 +156,15 @@ function reset_demo() {
         ./loaddata.py
     fi
 }
+
+function build_redwood() {
+    # Create a debug build of redwood and install it in the app-code virtualenv
+    # TODO: add live reload support
+    # We tell cargo to put its registry/crate cache and build cache in the target/ folder so
+    # it persists instead of being recreated from scratch each time the container starts.
+    # n.b. we can't re-use the host's caches because of permission differences
+    PATH="$PATH:/opt/cargo/bin/" \
+    CARGO_TARGET_DIR="${REPOROOT}/target/dev" CARGO_HOME="${REPOROOT}/target/cargo-dev" \
+        /opt/venvs/securedrop-build/bin/maturin build -m "${REPOROOT}/redwood/Cargo.toml"
+    /opt/venvs/securedrop-app-code/bin/pip install "${REPOROOT}"/target/dev/wheels/redwood-*.whl
+}
diff --git a/securedrop/bin/generate-docs-screenshots b/securedrop/bin/generate-docs-screenshots
@@ -10,6 +10,7 @@ run_tor &
 run_redis &
 run_x11vnc &
 urandom
+build_redwood
 maybe_create_config_py
 
 ./i18n_tool.py translate-messages --compile

diff --git a/securedrop/bin/run b/securedrop/bin/run
@@ -14,6 +14,7 @@ source "${BASH_SOURCE%/*}/dev-deps"
 
 run_redis &
 urandom
+build_redwood
 maybe_create_config_py
 reset_demo
 maybe_use_tor

diff --git a/securedrop/bin/run-mypy b/securedrop/bin/run-mypy
@@ -7,6 +7,9 @@ REPOROOT=$(git rev-parse --show-toplevel)
 cd "${REPOROOT}"
 
 if [ "$(command -v mypy)" ]; then
+    source "${BASH_SOURCE%/*}/dev-deps"
+    build_redwood
+
     mypy ./securedrop ./admin --namespace-packages --explicit-package-bases "$@"
 elif [ -d "/opt/venvs/securedrop-app-code/" ]; then
     # Inside the dev container, but no mypy

diff --git a/securedrop/bin/run-test b/securedrop/bin/run-test
@@ -24,6 +24,7 @@ run_redis &
 setup_vncauth
 run_x11vnc &
 urandom
+build_redwood
 maybe_create_config_py
 
 if [ -n "${CIRCLE_BRANCH:-}" ] ; then

diff --git a/securedrop/bin/translation-test b/securedrop/bin/translation-test
@@ -11,6 +11,7 @@ run_redis &
 setup_vncauth
 run_x11vnc &
 urandom
+build_redwood
 maybe_create_config_py
 ./i18n_tool.py translate-messages --compile
 

diff --git a/securedrop/dockerfiles/focal/python3/SlimDockerfile b/securedrop/dockerfiles/focal/python3/SlimDockerfile
@@ -9,16 +9,34 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get install
                        apache2-dev coreutils vim \
                        python3-pip python3-all python3-venv virtualenv python3-dev libssl-dev \
                        gnupg2 redis-server git curl wget \
-                       enchant libffi-dev sqlite3 gettext sudo tor basez
+                       enchant libffi-dev sqlite3 gettext sudo tor basez pkg-config
+
+# Install Rust
+ENV RUST_VERSION 1.69.0
+ENV RUSTUP_VERSION 1.24.3
+ENV RUSTUP_INIT_SHA256 3dc5ef50861ee18657f9db2eeb7392f9c2a6c95c90ab41e45ab4ca71476b4338
+ENV RUSTUP_HOME /opt/rustup
+ENV CARGO_HOME /opt/cargo
+
+RUN TMPDIR=`mktemp -d` && cd ${TMPDIR} \
+        && curl --proto '=https' --tlsv1.2 -OO -sSf https://static.rust-lang.org/rustup/archive/${RUSTUP_VERSION}/x86_64-unknown-linux-gnu/rustup-init \
+        && echo "${RUSTUP_INIT_SHA256} *rustup-init" | sha256sum -c - \
+        && chmod +x rustup-init \
+        && ./rustup-init --default-toolchain=${RUST_VERSION} --profile minimal -y \
+        && cd && rm -rf ${TMPDIR}
+# TODO: Remove after we switch to 1.70.0 when it's enabled by default
+ENV CARGO_REGISTRIES_CRATES_IO_PROTOCOL sparse
 
 COPY requirements requirements
+RUN python3 -m venv /opt/venvs/securedrop-build && \
+    /opt/venvs/securedrop-build/bin/pip3 install --no-deps --require-hashes -r requirements/python3/build-requirements.txt
 RUN python3 -m venv /opt/venvs/securedrop-app-code && \
     /opt/venvs/securedrop-app-code/bin/pip3 install --no-deps --require-hashes -r requirements/python3/bootstrap-requirements.txt && \
     /opt/venvs/securedrop-app-code/bin/pip3 install --no-deps --require-hashes -r requirements/python3/test-requirements.txt && \
     /opt/venvs/securedrop-app-code/bin/pip3 install --no-deps --require-hashes -r requirements/python3/requirements.txt
 
 RUN if test $USER_NAME != root ; then useradd --no-create-home --home-dir /tmp --uid $USER_ID $USER_NAME && echo "$USER_NAME ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers ; fi && \
-    chown -R $USER_NAME.$USER_NAME /opt/venvs/securedrop-app-code/
+    chown -R $USER_NAME.$USER_NAME /opt/venvs/
 
 STOPSIGNAL SIGKILL
 

diff --git a/securedrop/requirements/python3/build-requirements.in b/securedrop/requirements/python3/build-requirements.in
@@ -0,0 +1 @@
+maturin>=1.0,<2.0
diff --git a/securedrop/requirements/python3/build-requirements.txt b/securedrop/requirements/python3/build-requirements.txt
@@ -0,0 +1,25 @@
+#
+# This file is autogenerated by pip-compile
+# To update, run:
+#
+#    pip-compile --allow-unsafe --generate-hashes --output-file=requirements/python3/build-requirements.txt requirements/python3/build-requirements.in
+#
+maturin==1.0.1 \
+    --hash=sha256:04c0279dd0d6ccd317018bd1a43f52cbda715822537ae1a68015c9171f18b2fd \
+    --hash=sha256:10097e2602330c0b9db16d7dfd002476f5e5cf99df58ba2f3abc6de64a69e9a6 \
+    --hash=sha256:2907b345186a83db4bbe5571830509b3031784d08958b32d2ffa7857bd473725 \
+    --hash=sha256:6b020b9abbd1e9fef468c171216dc4be053834b5bf638075264ee090a993b0b0 \
+    --hash=sha256:6d9b4ff7c2d501e91886b859296f5c0478fc08bc7d537a72f98a69d51ff4f519 \
+    --hash=sha256:71fdb2dbbd5bcc60bd91ddcbe34dba9f04cc53c2add089a95a79d0d8fc8337b8 \
+    --hash=sha256:787bb56c80eda482ece2dd4788d479dbd0e74d981b2e2c538228365c19290fb7 \
+    --hash=sha256:8d88d1595d7514c27df96d5f4fe3dc5f24288528a746439403f27c3b448fca16 \
+    --hash=sha256:9ecebccb111c9c870fb2f5eee17518fe106f676227bb16f204a51e7a162aceec \
+    --hash=sha256:b39f9a42b3c8242e3f3ab990bd03ba989c6c07e4de9e21fcf877a2418119d445 \
+    --hash=sha256:c0b1efa47f8b7d15bc5945159764ce57316f9d1bfb7c8caa07cebdd41318359b \
+    --hash=sha256:d271b24febbfc020561984b1acdfc39b132df21f4e42d7af0fe274ea738c8000 \
+    --hash=sha256:d392ec0578d9e6f03914837cef7bbb264d5708807e0b48176b6ff0b50083ba7c
+    # via -r requirements/python3/build-requirements.in
+tomli==2.0.1 \
+    --hash=sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc \
+    --hash=sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f
+    # via maturin