Add task to cache cargo crates and use it for MacOS/Windows builds (#…

…9841)

* Add task to cache cargo crates and use it for MacOS builds
* Bug: Taskcluster transforms fail if the toolchain has no env
* Windows: Use crate toolchain and skip build/source
* Windows: remove unnecessary dep on build/source
* Convert WIREGUARD_FOLDER to CMake paths

taskcluster/kinds/build/macos.yml

@@ -13,6 +13,7 @@ task-defaults:
  worker-type: b-macos
  fetches:
  toolchain:
+ - cargo-vendor
  - conda-macos
  worker:
  taskcluster-proxy: true

taskcluster/kinds/build/windows.yml

@@ -10,18 +10,15 @@ task-defaults:
  tier: 1
  platform: windows/x86_64
  worker-type: b-win2022
- dependencies:
- build: build-source/vpn
  fetches:
  fetch:
  - win-dev-env
  - win-sentry-cli
  - win-conda
  - wireguard-nt
  toolchain:
+ - cargo-vendor
  - conda-windows-x86_64
- build:
- - artifact: mozillavpn-sources.tar.gz
  worker:
  taskcluster-proxy: true
  chain-of-trust: true

taskcluster/kinds/toolchain/cargo.yml

@@ -0,0 +1,20 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+---
+cargo-vendor:
+ description: "Vendored Rust Crates"
+ worker-type: b-linux
+ worker:
+ env:
+ FOO: bar # Bug: taskcluster fails if there is no env key?
+ max-run-time: 3600
+ docker-image: {in-tree: build}
+ treeherder:
+ symbol: TL(cargo-vendor)
+ run:
+ script: cargo_vendor_crates.sh
+ resources:
+ - Cargo.lock
+ toolchain-alias: cargo-vendor
+ toolchain-artifact: public/build/cargo-vendor.tar.xz

taskcluster/kinds/toolchain/kind.yml

@@ -29,3 +29,4 @@ tasks-from:
  - conda_windows.yml
  - conda_osx.yml
  - flatpak.yml
+ - cargo.yml

taskcluster/scripts/build/macos_build.sh

@@ -73,6 +73,18 @@ then
  npm install -g @sentry/cli
 fi
 
+# Use vendored crates - if available.
+if [ -d ${MOZ_FETCHES_DIR}/cargo-vendor ]; then
+mkdir -p .cargo
+cat << EOF > .cargo/config.toml
+[source.vendored-sources]
+directory = "${MOZ_FETCHES_DIR}/cargo-vendor"
+
+[source.crates-io]
+replace-with = "vendored-sources"
+EOF
+fi
+
 print Y "Configuring the build..."
 if [ -d ${TASK_HOME}/build ]; then
  echo "Found old build-folder, weird!"

taskcluster/scripts/build/windows_clang_cl.ps1

@@ -13,6 +13,11 @@ Set-Location -Path $TASK_WORKDIR
 . "$FETCHES_PATH/QT_OUT/configure_qt.ps1"
 
 
+# Ensure we are working from a full checkout of all submodules. Taskcluster
+# doesn't handle recursive submodules, so we'll need to do it ourselves.
+git -C "$REPO_ROOT_PATH" submodule update --init --recursive
+
+
 # We have not yet removed our VC_Redist strategy. Therefore we rely on the old vsstudio bundle to get us that :)
 # TODO: We need to handle this at some point.
 $env:VCToolsRedistDir=(resolve-path "$FETCHES_PATH/VisualStudio/VC/Redist/MSVC/14.30.30704/").ToString()
@@ -31,15 +36,22 @@ if (Test-Path -Path $SSL_PATH) {
 }
 
 
-# Extract the sources
-$SOURCE_DSC = resolve-path "$FETCHES_PATH/mozillavpn_*.dsc"
-$SOURCE_VERSION = ((select-string $SOURCE_DSC -Pattern '^Version:') -split " ")[1]
-tar -xzvf (resolve-path "$FETCHES_PATH/mozillavpn_$SOURCE_VERSION.orig.tar.gz" -Relative)
-$SOURCE_DIR = resolve-path "$TASK_WORKDIR/mozillavpn-$SOURCE_VERSION"
+## Use vendored rust crates, if present
+if (Test-Path -Path "$FETCHES_PATH\cargo-vendor") {
+ $CARGO_VENDOR_PATH = "$FETCHES_PATH/cargo-vendor" -replace @('\\', '/')
+ New-Item -Path "$REPO_ROOT_PATH\.cargo" -ItemType "directory" -Force
+@"
+[source.vendored-sources]
+directory = "$CARGO_VENDOR_PATH"
+
+[source.crates-io]
+replace-with = "vendored-sources"
+"@ | Out-File -Encoding utf8 $REPO_ROOT_PATH\.cargo\config.toml
+}
 
 
 ## Setup the conda environment
-. $SOURCE_DIR/scripts/utils/call_bat.ps1 $FETCHES_PATH/Scripts/activate.bat
+. $REPO_ROOT_PATH/scripts/utils/call_bat.ps1 $FETCHES_PATH/Scripts/activate.bat
 conda-unpack
 
 # Conda Pack excpets to be run under cmd. therefore it will
@@ -65,7 +77,7 @@ mkdir $TASK_WORKDIR/cmake_build
 $BUILD_DIR =resolve-path "$TASK_WORKDIR/cmake_build"
 
 # Do the generic build
-cmake -S $SOURCE_DIR -B $BUILD_DIR -GNinja `
+cmake -S $REPO_ROOT_PATH -B $BUILD_DIR -GNinja `
  -DCMAKE_BUILD_TYPE=Release `
  -DPYTHON_EXECUTABLE="$CONDA_PREFIX\python.exe" `
  -DGOLANG_BUILD_TOOL="$CONDA_PREFIX\bin\go.exe" `
@@ -91,7 +103,7 @@ Get-ChildItem -Path $TASK_WORKDIR/artifacts
 
 
 Get-command python
-python $SOURCE_DIR/taskcluster/scripts/get-secret.py -s project/mozillavpn/level-1/sentry -k sentry_debug_file_upload_key -f sentry_debug_file_upload_key
+python $REPO_ROOT_PATH/taskcluster/scripts/get-secret.py -s project/mozillavpn/level-1/sentry -k sentry_debug_file_upload_key -f sentry_debug_file_upload_key
 $env:SENTRY_AUTH_TOKEN=$(Get-Content sentry_debug_file_upload_key)
 # Are we logged in?
 sentry-cli-Windows-x86_64.exe info

taskcluster/scripts/toolchain/cargo_vendor_crates.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+set -e
+
+cargo vendor --manifest-path $VCS_PATH/Cargo.toml $(pwd)/cargo-vendor
+tar -cJf $UPLOAD_DIR/cargo-vendor.tar.xz cargo-vendor/

windows/wireguard_nt/CMakeLists.txt

@@ -2,9 +2,6 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-## Download and extract Wix and friends.
-
-
 set(WIREGUARD_FOLDER $ENV{WIREGUARD_FOLDER} CACHE STRING "Path to Extracted WireGuardNT Folder")
 
 if(NOT WIREGUARD_FOLDER)
@@ -14,17 +11,19 @@ if(NOT WIREGUARD_FOLDER)
 
  file(ARCHIVE_EXTRACT INPUT ${CMAKE_CURRENT_BINARY_DIR}/.deps/wg-nt.zip DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/.deps)
 
- set(WIREGUARD_FOLDER ${CMAKE_CURRENT_BINARY_DIR}/.deps/wireguard-nt) 
+ set(CM_WIREGUARD_FOLDER ${CMAKE_CURRENT_BINARY_DIR}/.deps/wireguard-nt) 
+else()
+ cmake_path(CONVERT ${WIREGUARD_FOLDER} TO_CMAKE_PATH_LIST CM_WIREGUARD_FOLDER)
 endif()
 
 
 
-include_directories(${WIREGUARD_FOLDER}/include)
+include_directories(${CM_WIREGUARD_FOLDER}/include)
 
 install(FILES
- ${WIREGUARD_FOLDER}/bin/${CMAKE_SYSTEM_PROCESSOR}/wireguard.dll
+ ${CM_WIREGUARD_FOLDER}/bin/${CMAKE_SYSTEM_PROCESSOR}/wireguard.dll
  DESTINATION .
 )
 
-target_include_directories(mozillavpn PUBLIC ${WIREGUARD_FOLDER}/include)
-target_include_directories(shared-sources INTERFACE ${WIREGUARD_FOLDER}/include)
+target_include_directories(mozillavpn PUBLIC ${CM_WIREGUARD_FOLDER}/include)
+target_include_directories(shared-sources INTERFACE ${CM_WIREGUARD_FOLDER}/include)