Pregeneration of NASM build artifacts (#491)

* Allow build with prebuilt NASM objects * Symbols from ubuntu-latest * Symbols from macos-13-xlarge * Symbols from x86_64-pc-windows-msvc * Symbols from x86_64-pc-windows-gnu * Symbols from i686-pc-windows-msvc * Symbols from macos-12 * Symbols from aarch64-pc-windows-msvc * Symbols for powerpc-unknown-linux-gnu * Symbols for armv7-unknown-linux-gnueabihf * Symbols for i686-unknown-linux-gnu * Symbols for arm-unknown-linux-gnueabi * Symbols for x86_64-unknown-linux-musl * Symbols for powerpc64le-unknown-linux-gnu * Symbols for aarch64-unknown-linux-gnu * Symbols for aarch64-unknown-linux-musl * Generated headers * Generated bindings from ubuntu-latest * Generated bindings from macos-13-xlarge * Generated bindings for x86_64-pc-windows-gnu * Generated bindings for i686-pc-windows-msvc * Generated bindings from macos-12 * Generated bindings for aarch64-pc-windows-msvc * Generated bindings for x86_64-pc-windows-msvc * Generated bindings for aarch64-unknown-linux-gnu * Generated bindings for x86_64-unknown-linux-musl * Generated bindings for aarch64-unknown-linux-musl * Generated bindings for i686-unknown-linux-gnu * Collected source files from ubuntu-latest * Collected source files from macos-13-xlarge * Collected source files from macos-12 * Collected NASM files * Collected source files for x86_64-unknown-linux-musl * Collected source files for i686-unknown-linux-gnu * Collected source files for aarch64-unknown-linux-gnu * Collected source files for aarch64-unknown-linux-musl * Add copyright to prebuilt-nasm.bat --------- Co-authored-by: Justin Smith <justin.w.smith@gmail.com> Co-authored-by: aws-lc-sys-bindings-generator <aws-lc-github-bot@amazon.com>
aws · Aug 27, 2024 · f4e09c2 · f4e09c2
1 parent 00ac18e
commit f4e09c2
Show file tree

Hide file tree

Showing 59 changed files with 63,760 additions and 44 deletions.
diff --git a/.github/workflows/sys-bindings-generator.yml b/.github/workflows/sys-bindings-generator.yml
@@ -270,3 +270,36 @@ jobs:
         run: ./scripts/build/collect_build_src.sh -t ${{ matrix.target }}
       - name: Commit & Push changes
         run: ./scripts/ci/ci_add_commit_rebase_push.sh "Collected source files for ${{ matrix.target }}"
+  collect-nasm-and-commit:
+    needs: generate-windows-bindings-and-commit
+    if: github.repository == 'aws/aws-lc-rs'
+    runs-on: windows-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: 'recursive'
+          ref: ${{ github.ref_name }}
+      - uses: dtolnay/rust-toolchain@master
+        id: toolchain
+        with:
+          toolchain: stable
+          targets: "x86_64-pc-windows-msvc,x86_64-pc-windows-gnu"
+      - uses: ilammy/setup-nasm@v1
+      - name: Build aws-lc-sys
+        shell: bash
+        run: AWS_LC_SYS_PREBUILT_NASM=0 cargo build -p aws-lc-sys --release --target x86_64-pc-windows-msvc
+      - name: Collect NASM object files
+        shell: bash
+        run: ./scripts/build/collect_nasm_obj.sh
+      - name: Clean build
+        shell: bash
+        run: cargo clean
+      - name: Test aws-lc-rs for x86_64-pc-windows-msvc
+        shell: bash
+        run: AWS_LC_SYS_PREBUILT_NASM=1 cargo build -p aws-lc-sys --target x86_64-pc-windows-msvc
+      - name: Test aws-lc-sys for x86_64-pc-windows-gnu
+        shell: bash
+        run: AWS_LC_SYS_PREBUILT_NASM=1 cargo build -p aws-lc-sys --target x86_64-pc-windows-gnu
+      - name: Commit & Push changes
+        shell: bash
+        run: ./scripts/ci/ci_add_commit_rebase_push.sh "Collected NASM files"
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -37,7 +37,7 @@ jobs:
           - --no-default-features --features non-fips,ring-sig-verify,unstable
           - --no-default-features --features non-fips,alloc,unstable
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           submodules: 'recursive'
       - uses: dtolnay/rust-toolchain@master
@@ -66,7 +66,7 @@ jobs:
           - --no-default-features --features aws-lc-sys,bindgen,unstable
           - --release --all-targets --features bindgen,unstable
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           submodules: 'recursive'
       - if: ${{ matrix.os == 'macos-13-xlarge' }}
@@ -106,9 +106,10 @@ jobs:
           - --no-default-features --features non-fips,ring-io,unstable
           - --no-default-features --features non-fips,ring-sig-verify,unstable
           - --no-default-features --features non-fips,alloc,unstable
+    env:
+      AWS_LC_SYS_PREBUILT_NASM: 1
     steps:
-      - uses: ilammy/setup-nasm@v1
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           submodules: 'recursive'
       - uses: dtolnay/rust-toolchain@master
@@ -125,7 +126,7 @@ jobs:
     name: aws-ls-rs coverage
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           submodules: 'recursive'
           lfs: true
@@ -167,7 +168,7 @@ jobs:
           - --no-default-features --features fips,asan
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           submodules: 'recursive'
       - uses: dtolnay/rust-toolchain@master
@@ -196,7 +197,7 @@ jobs:
         os: [ ubuntu-latest, macos-12, macos-13-xlarge ]
         static: [ 0, 1 ]
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           submodules: 'recursive'
       - uses: dtolnay/rust-toolchain@stable
@@ -205,6 +206,40 @@ jobs:
         # See: https://github.com/rust-lang/cargo/issues/8531
         run: cargo test -p aws-lc-rs --tests
 
+  build-env-nasm-test:
+    if: github.repository_owner == 'aws'
+    name: prebuilt NASM verification
+    runs-on: windows-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        target:
+          - 'x86_64-pc-windows-msvc'
+          - 'x86_64-pc-windows-gnu'
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: 'recursive'
+      - uses: dtolnay/rust-toolchain@stable
+      - name: Install NASM
+        uses: ilammy/setup-nasm@v1
+      - name: Remove NASM artifacts
+        shell: bash
+        run: |
+          cargo clean
+          rm ./aws-lc-sys/builder/prebuilt-nasm/*
+      - name: Run cargo test
+        shell: bash
+        run: AWS_LC_SYS_PREBUILT_NASM=0 cargo test --tests -p aws-lc-rs --release --no-default-features --features aws-lc-sys
+      - name: Collect NASM outputs
+        shell: bash
+        run: ./scripts/build/collect_nasm_obj.sh
+      - name: Flag any NASM changes
+        shell: bash
+        run: |
+          git add .
+          git diff --cached --exit-code HEAD -- aws-lc-sys/builder/prebuilt-nasm/*.txt
+
   build-env-external-bindgen-test:
     if: github.repository_owner == 'aws'
     name: aws-lc-rs FIPS - External bindgen test
@@ -216,14 +251,14 @@ jobs:
       matrix:
         os: [ ubuntu-latest, macos-12, macos-13-xlarge, windows-latest ]
     steps:
-      - if: ${{ matrix.os == 'windows-latest' }}
-        uses: ilammy/setup-nasm@v1
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           submodules: 'recursive'
       - uses: dtolnay/rust-toolchain@stable
-      - name: Install ninja-build tool
-        uses: seanmiddleditch/gha-setup-ninja@v4
+      - if: ${{ matrix.os == 'windows-latest' }}
+        uses: ilammy/setup-nasm@v1
+      - if: ${{ matrix.os == 'windows-latest' }}
+        uses: seanmiddleditch/gha-setup-ninja@v5
       - name: Install bindgen-cli
         run: cargo install --locked bindgen-cli
       - name: Remove bindings
@@ -245,12 +280,10 @@ jobs:
         os: [ ubuntu-latest, macos-12, macos-13-xlarge ]
         static: [ 0, 1 ]
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           submodules: 'recursive'
       - uses: dtolnay/rust-toolchain@stable
-      - name: Install ninja-build tool
-        uses: seanmiddleditch/gha-setup-ninja@v4
       - uses: actions/setup-go@v4
         with:
           go-version: '>=1.18'
@@ -271,7 +304,7 @@ jobs:
       matrix:
         os: [ ubuntu-latest, macos-12, macos-13-xlarge, windows-latest ]
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           submodules: 'recursive'
       - uses: dtolnay/rust-toolchain@stable
@@ -306,15 +339,14 @@ jobs:
       matrix:
         os: [ ubuntu-latest, macos-12, macos-13-xlarge, windows-latest ]
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           submodules: 'recursive'
       - uses: dtolnay/rust-toolchain@stable
-      - name: Install ninja-build tool
-        uses: seanmiddleditch/gha-setup-ninja@v4
       - uses: actions/setup-go@v4
         with:
           go-version: '>=1.18'
+      - uses: seanmiddleditch/gha-setup-ninja@v5
       - name: Run cargo test
         run: cargo test -p aws-lc-rs --tests --no-default-features --features fips
       - name: Release build
@@ -346,14 +378,14 @@ jobs:
       matrix:
         os: [ ubuntu-latest, macos-12, macos-13-xlarge, windows-latest ]
     steps:
-      - if: ${{ matrix.os == 'windows-latest' }}
-        uses: ilammy/setup-nasm@v1
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           submodules: 'recursive'
+      - if: ${{ matrix.os == 'windows-latest' }}
+        uses: ilammy/setup-nasm@v1
+      - if: ${{ matrix.os == 'windows-latest' }}
+        uses: seanmiddleditch/gha-setup-ninja@v5
       - uses: dtolnay/rust-toolchain@stable
-      - name: Install ninja-build tool
-        uses: seanmiddleditch/gha-setup-ninja@v4
       - uses: actions/setup-go@v4
         with:
           go-version: '>=1.18'
@@ -378,7 +410,7 @@ jobs:
           - macos-12
           - macos-13-xlarge
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           submodules: 'recursive'
           lfs: true

diff --git a/aws-lc-sys/builder/cc_builder/aarch64_apple_darwin.rs b/aws-lc-sys/builder/cc_builder/aarch64_apple_darwin.rs
@@ -1,6 +1,6 @@
 // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 // SPDX-License-Identifier: Apache-2.0 OR ISC
-// Wed Aug 21 14:33:51 UTC 2024
+// Wed Aug 21 22:12:32 UTC 2024
 
 use crate::cc_builder::Library;
 

diff --git a/aws-lc-sys/builder/cc_builder/aarch64_unknown_linux_gnu.rs b/aws-lc-sys/builder/cc_builder/aarch64_unknown_linux_gnu.rs
@@ -1,6 +1,6 @@
 // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 // SPDX-License-Identifier: Apache-2.0 OR ISC
-// Wed Aug 21 14:39:05 UTC 2024
+// Wed Aug 21 22:18:30 UTC 2024
 
 use crate::cc_builder::Library;
 

diff --git a/aws-lc-sys/builder/cc_builder/aarch64_unknown_linux_musl.rs b/aws-lc-sys/builder/cc_builder/aarch64_unknown_linux_musl.rs
@@ -1,6 +1,6 @@
 // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 // SPDX-License-Identifier: Apache-2.0 OR ISC
-// Wed Aug 21 14:40:07 UTC 2024
+// Wed Aug 21 22:18:54 UTC 2024
 
 use crate::cc_builder::Library;
 

diff --git a/aws-lc-sys/builder/cc_builder/i686_unknown_linux_gnu.rs b/aws-lc-sys/builder/cc_builder/i686_unknown_linux_gnu.rs
@@ -1,6 +1,6 @@
 // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 // SPDX-License-Identifier: Apache-2.0 OR ISC
-// Wed Aug 21 14:39:33 UTC 2024
+// Wed Aug 21 22:19:08 UTC 2024
 
 use crate::cc_builder::Library;
 

diff --git a/aws-lc-sys/builder/cc_builder/x86_64_apple_darwin.rs b/aws-lc-sys/builder/cc_builder/x86_64_apple_darwin.rs
@@ -1,6 +1,6 @@
 // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 // SPDX-License-Identifier: Apache-2.0 OR ISC
-// Wed Aug 21 14:35:11 UTC 2024
+// Wed Aug 21 22:13:39 UTC 2024
 
 use crate::cc_builder::Library;
 

diff --git a/aws-lc-sys/builder/cc_builder/x86_64_unknown_linux_gnu.rs b/aws-lc-sys/builder/cc_builder/x86_64_unknown_linux_gnu.rs
@@ -1,6 +1,6 @@
 // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 // SPDX-License-Identifier: Apache-2.0 OR ISC
-// Wed Aug 21 14:33:41 UTC 2024
+// Wed Aug 21 22:12:14 UTC 2024
 
 use crate::cc_builder::Library;
 

diff --git a/aws-lc-sys/builder/cc_builder/x86_64_unknown_linux_musl.rs b/aws-lc-sys/builder/cc_builder/x86_64_unknown_linux_musl.rs
@@ -1,6 +1,6 @@
 // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 // SPDX-License-Identifier: Apache-2.0 OR ISC
-// Wed Aug 21 14:40:14 UTC 2024
+// Wed Aug 21 22:18:39 UTC 2024
 
 use crate::cc_builder::Library;
 

diff --git a/aws-lc-sys/builder/cmake_builder.rs b/aws-lc-sys/builder/cmake_builder.rs
@@ -3,9 +3,9 @@
 
 use crate::OutputLib::{Crypto, RustWrapper, Ssl};
 use crate::{
-    cargo_env, emit_warning, execute_command, is_crt_static, is_no_asm, option_env, target,
-    target_arch, target_env, target_family, target_os, target_underscored, target_vendor,
-    OutputLibType,
+    allow_prebuilt_nasm, cargo_env, emit_warning, execute_command, is_crt_static, is_no_asm,
+    option_env, target, target_arch, target_env, target_family, target_os, target_underscored,
+    target_vendor, test_nasm_command, OutputLibType,
 };
 use std::env;
 use std::ffi::OsString;
@@ -22,10 +22,6 @@ fn test_clang_cl_command() -> bool {
     execute_command("clang-cl".as_ref(), &["--version".as_ref()]).status
 }
 
-fn test_nasm_command() -> bool {
-    execute_command("nasm".as_ref(), &["-version".as_ref()]).status
-}
-
 fn find_cmake_command() -> Option<OsString> {
     if let Some(cmake) = option_env("CMAKE") {
         emit_warning(&format!(
@@ -162,7 +158,7 @@ impl CmakeBuilder {
 
         // See issue: https://github.com/aws/aws-lc-rs/issues/453
         if target_os() == "windows" {
-            Self::configure_windows(&mut cmake_cfg);
+            self.configure_windows(&mut cmake_cfg);
         }
 
         // If the build environment vendor is Apple
@@ -213,7 +209,7 @@ impl CmakeBuilder {
         cmake_cfg
     }
 
-    fn configure_windows(cmake_cfg: &mut cmake::Config) {
+    fn configure_windows(&self, cmake_cfg: &mut cmake::Config) {
         match (target_env().as_str(), target_arch().as_str()) {
             ("msvc", "aarch64") => {
                 cmake_cfg.generator_toolset(format!(
@@ -243,6 +239,23 @@ impl CmakeBuilder {
             }
             _ => {}
         }
+        if target_arch() == "x86_64" && !test_nasm_command() && Some(true) == allow_prebuilt_nasm()
+        {
+            emit_warning("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!");
+            emit_warning("!!!   Using pre-built NASM binaries   !!!");
+            emit_warning("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!");
+
+            let script_path = self
+                .manifest_dir
+                .join("builder")
+                .join("prebuilt-nasm.bat")
+                .display()
+                .to_string();
+            let script_path = script_path.replace('\\', "/");
+
+            cmake_cfg.define("CMAKE_ASM_NASM_COMPILER", script_path.as_str());
+            cmake_cfg.define("CMAKE_VERBOSE_MAKEFILE", "1");
+        }
     }
 
     fn configure_open_harmony(cmake_cfg: &mut cmake::Config) {
@@ -298,10 +311,14 @@ impl crate::Builder for CmakeBuilder {
         let mut missing_dependency = false;
 
         if target_os() == "windows" {
-            if target_arch() == "x86_64" && !test_nasm_command() && !is_no_asm() {
+            if target_arch() == "x86_64"
+                && !is_no_asm()
+                && !test_nasm_command()
+                && Some(true) != allow_prebuilt_nasm()
+            {
                 eprintln!(
-                    "Consider setting `AWS_LC_SYS_NO_ASM` in the environment for development builds.\
-                See User Guide about the limitations: https://aws.github.io/aws-lc-rs/index.html"
+                    "Consider setting `AWS_LC_SYS_PREBUILT_NASM` in the build environment.\
+                See User Guide: https://aws.github.io/aws-lc-rs/index.html"
                 );
                 eprintln!("Missing dependency: nasm");
                 missing_dependency = true;