From 8a5dab47b8591a8a161dc22f2c4a20980c6f68d4 Mon Sep 17 00:00:00 2001 From: Tony Arcieri Date: Sat, 6 Jan 2024 11:43:36 -0800 Subject: [PATCH] aes: replace inline ASM with ARMv8 intrinsics (#380) Note: bumps the MSRV for `aes_armv8` from 1.65 -> 1.72 Rust 1.72 stabilized the ARMv8 AES intrinsics, which means we no longer need to use inline `asm!` "polyfills" for these functions to support stable Rust. --- .github/workflows/aes.yml | 2 +- aes/src/armv8.rs | 1 - aes/src/armv8/encdec.rs | 34 ++++++++------ aes/src/armv8/expand.rs | 4 -- aes/src/armv8/hazmat.rs | 3 -- aes/src/armv8/intrinsics.rs | 93 ------------------------------------- 6 files changed, 21 insertions(+), 116 deletions(-) delete mode 100644 aes/src/armv8/intrinsics.rs diff --git a/.github/workflows/aes.yml b/.github/workflows/aes.yml index 33973467..a8ab83d8 100644 --- a/.github/workflows/aes.yml +++ b/.github/workflows/aes.yml @@ -217,7 +217,7 @@ jobs: matrix: include: - target: aarch64-unknown-linux-gnu - rust: 1.65.0 # MSRV + rust: 1.72.0 # MSRV for `aes_armv8` runs-on: ubuntu-latest # Cross mounts only current package, i.e. by default it ignores workspace's Cargo.toml defaults: diff --git a/aes/src/armv8.rs b/aes/src/armv8.rs index 20307bb9..826dfd04 100644 --- a/aes/src/armv8.rs +++ b/aes/src/armv8.rs @@ -14,7 +14,6 @@ pub(crate) mod hazmat; mod encdec; mod expand; -mod intrinsics; #[cfg(test)] mod test_expand; diff --git a/aes/src/armv8/encdec.rs b/aes/src/armv8/encdec.rs index 09c59cee..7f462564 100644 --- a/aes/src/armv8/encdec.rs +++ b/aes/src/armv8/encdec.rs @@ -4,12 +4,6 @@ use crate::{Block, Block8}; use cipher::inout::InOut; use core::arch::aarch64::*; -// Stable "polyfills" for unstable core::arch::aarch64 intrinsics -// TODO(tarcieri): remove when these intrinsics have been stabilized -use super::intrinsics::{ - vaesdq_u8, vaesdq_u8_and_vaesimcq_u8, vaeseq_u8, vaeseq_u8_and_vaesmcq_u8, -}; - /// Perform AES encryption using the given expanded keys. #[target_feature(enable = "aes")] #[target_feature(enable = "neon")] @@ -25,8 +19,11 @@ pub(super) unsafe fn encrypt1( let mut state = vld1q_u8(in_ptr as *const u8); for k in expanded_keys.iter().take(rounds - 1) { - // AES single round encryption and mix columns - state = vaeseq_u8_and_vaesmcq_u8(state, *k); + // AES single round encryption + state = vaeseq_u8(state, *k); + + // Mix columns + state = vaesmcq_u8(state); } // AES single round encryption @@ -65,8 +62,11 @@ pub(super) unsafe fn encrypt8( for k in expanded_keys.iter().take(rounds - 1) { for i in 0..8 { - // AES single round encryption and mix columns - state[i] = vaeseq_u8_and_vaesmcq_u8(state[i], *k); + // AES single round encryption + state[i] = vaeseq_u8(state[i], *k); + + // Mix columns + state[i] = vaesmcq_u8(state[i]); } } @@ -95,8 +95,11 @@ pub(super) unsafe fn decrypt1( let mut state = vld1q_u8(in_ptr as *const u8); for k in expanded_keys.iter().take(rounds - 1) { - // AES single round decryption and inverse mix columns - state = vaesdq_u8_and_vaesimcq_u8(state, *k); + // AES single round decryption + state = vaesdq_u8(state, *k); + + // Inverse mix columns + state = vaesimcq_u8(state); } // AES single round decryption @@ -135,8 +138,11 @@ pub(super) unsafe fn decrypt8( for k in expanded_keys.iter().take(rounds - 1) { for i in 0..8 { - // AES single round decryption and inverse mix columns - state[i] = vaesdq_u8_and_vaesimcq_u8(state[i], *k); + // AES single round decryption + state[i] = vaesdq_u8(state[i], *k); + + // Inverse mix columns + state[i] = vaesimcq_u8(state[i]); } } diff --git a/aes/src/armv8/expand.rs b/aes/src/armv8/expand.rs index 0cfc52d1..0a0eaa67 100644 --- a/aes/src/armv8/expand.rs +++ b/aes/src/armv8/expand.rs @@ -2,10 +2,6 @@ use core::{arch::aarch64::*, mem, slice}; -// Stable "polyfills" for unstable core::arch::aarch64 intrinsics -// TODO(tarcieri): remove when these intrinsics have been stabilized -use super::intrinsics::{vaeseq_u8, vaesimcq_u8}; - /// There are 4 AES words in a block. const BLOCK_WORDS: usize = 4; diff --git a/aes/src/armv8/hazmat.rs b/aes/src/armv8/hazmat.rs index 3e078cfe..f094243c 100644 --- a/aes/src/armv8/hazmat.rs +++ b/aes/src/armv8/hazmat.rs @@ -7,9 +7,6 @@ use crate::{Block, Block8}; use core::arch::aarch64::*; -// Stable "polyfills" for unstable core::arch::aarch64 intrinsics -use super::intrinsics::{vaesdq_u8, vaeseq_u8, vaesimcq_u8, vaesmcq_u8}; - /// AES cipher (encrypt) round function. #[allow(clippy::cast_ptr_alignment)] #[target_feature(enable = "aes")] diff --git a/aes/src/armv8/intrinsics.rs b/aes/src/armv8/intrinsics.rs deleted file mode 100644 index 752af492..00000000 --- a/aes/src/armv8/intrinsics.rs +++ /dev/null @@ -1,93 +0,0 @@ -//! Stable "polyfills" for unstable `core::arch::aarch64` intrinsics which use -//! `asm!` internally to allow use on stable Rust. -// TODO(tarcieri): remove when these intrinsics have been stabilized - -use core::arch::{aarch64::uint8x16_t, asm}; - -/// AES single round encryption. -#[inline] -#[target_feature(enable = "aes")] -pub(super) unsafe fn vaeseq_u8(mut data: uint8x16_t, key: uint8x16_t) -> uint8x16_t { - asm!( - "AESE {d:v}.16B, {k:v}.16B", - d = inout(vreg) data, - k = in(vreg) key, - options(pure, nomem, nostack, preserves_flags) - ); - data -} - -/// AES single round decryption. -#[inline] -#[target_feature(enable = "aes")] -pub(super) unsafe fn vaesdq_u8(mut data: uint8x16_t, key: uint8x16_t) -> uint8x16_t { - asm!( - "AESD {d:v}.16B, {k:v}.16B", - d = inout(vreg) data, - k = in(vreg) key, - options(pure, nomem, nostack, preserves_flags) - ); - data -} - -/// AES mix columns. -#[cfg(feature = "hazmat")] -#[inline] -#[target_feature(enable = "aes")] -pub(super) unsafe fn vaesmcq_u8(mut data: uint8x16_t) -> uint8x16_t { - asm!( - "AESMC {d:v}.16B, {d:v}.16B", - d = inout(vreg) data, - options(pure, nomem, nostack, preserves_flags) - ); - data -} - -/// AES inverse mix columns. -#[inline] -#[target_feature(enable = "aes")] -pub(super) unsafe fn vaesimcq_u8(mut data: uint8x16_t) -> uint8x16_t { - asm!( - "AESIMC {d:v}.16B, {d:v}.16B", - d = inout(vreg) data, - options(pure, nomem, nostack, preserves_flags) - ); - data -} - -/// AES single round encryption combined with mix columns. -/// -/// These two instructions are combined into a single assembly block to ensure -/// that instructions fuse properly. -#[inline] -#[target_feature(enable = "aes")] -pub(super) unsafe fn vaeseq_u8_and_vaesmcq_u8(mut data: uint8x16_t, key: uint8x16_t) -> uint8x16_t { - asm!( - "AESE {d:v}.16B, {k:v}.16B", - "AESMC {d:v}.16B, {d:v}.16B", - d = inout(vreg) data, - k = in(vreg) key, - options(pure, nomem, nostack, preserves_flags) - ); - data -} - -/// AES single round decryption combined with mix columns. -/// -/// These two instructions are combined into a single assembly block to ensure -/// that instructions fuse properly. -#[inline] -#[target_feature(enable = "aes")] -pub(super) unsafe fn vaesdq_u8_and_vaesimcq_u8( - mut data: uint8x16_t, - key: uint8x16_t, -) -> uint8x16_t { - asm!( - "AESD {d:v}.16B, {k:v}.16B", - "AESIMC {d:v}.16B, {d:v}.16B", - d = inout(vreg) data, - k = in(vreg) key, - options(pure, nomem, nostack, preserves_flags) - ); - data -}