Emit error when calling/declaring functions with unavailable vectors.

On some architectures, vector types may have a different ABI when
relevant target features are enabled.

As discussed in rust-lang/lang-team#235, this
turns out to very easily lead to unsound code.

This commit makes it an error to declare or call functions using those
vector types in a context in which the corresponding target features are
disabled, if using an ABI for which the difference is relevant.

Cargo.lock

@@ -4139,6 +4139,7 @@ dependencies = [
 name = "rustc_monomorphize"
 version = "0.0.0"
 dependencies = [
+ "rustc_abi",
  "rustc_data_structures",
  "rustc_errors",
  "rustc_fluent_macro",

compiler/rustc_lint_defs/src/builtin.rs

@@ -16,6 +16,7 @@ declare_lint_pass! {
  /// that are used by other parts of the compiler.
  HardwiredLints => [
  // tidy-alphabetical-start
+ ABI_ERROR_DISABLED_VECTOR_TYPE,
  ABSOLUTE_PATHS_NOT_STARTING_WITH_CRATE,
  AMBIGUOUS_ASSOCIATED_ITEMS,
  AMBIGUOUS_GLOB_IMPORTS,
@@ -5078,3 +5079,43 @@ declare_lint! {
  };
  crate_level_only
 }
+
+declare_lint! {
+ /// The `abi_error_disabled_vector_type` lint detects function definitions and calls
+ /// whose ABI depends on enabling certain target features that do not enable those features.
+ ///
+ /// ###
+ ///
+ /// ```rust,compile_fail
+ /// #![deny(abi_error_disabled_vector_type)]
+ /// #![allow(improper_ctypes_definitions)] // false positive
+ ///
+ /// extern "C" fn foo(_: std::arch::x86_64::__m256) {
+ /// todo!()
+ /// }
+ ///
+ /// #[target_feature(enable = "avx2")]
+ /// unsafe extern "C" fn favx(_: std::arch::x86_64::__m256) {
+ /// todo!()
+ /// }
+ /// ```
+ ///
+ /// {{produces}}
+ ///
+ /// ### Explanation
+ ///
+ /// The ABI of `foo` is somewhat surprising: since AVX may not be enabled when compiling it,
+ /// the parameter may be passed by stack and not by register. This then easily leads to
+ /// undefined behaviour if calling the function from a function for which AVX is enabled.
+ /// A similar (but complementary) problem is triggered by a caller that does *not* enable
+ /// the AVX feature calling `favx`.
+ ///
+ /// Note that this lint is very similar to the `-Wpsabi` warning in `gcc`/`clang`.
+ pub ABI_ERROR_DISABLED_VECTOR_TYPE,
+ Warn,
+ "this function call or definition uses a vector type which is not enabled",
+ @future_incompatible = FutureIncompatibleInfo {
+ reason: FutureIncompatibilityReason::FutureReleaseErrorDontReportInDeps,
+ reference: "issue #116558 <https://github.com/rust-lang/rust/issues/116558>",
+ };
+}

compiler/rustc_monomorphize/Cargo.toml

@@ -5,6 +5,7 @@ edition = "2021"
 
 [dependencies]
 # tidy-alphabetical-start
+rustc_abi = { path = "../rustc_abi" }
 rustc_data_structures = { path = "../rustc_data_structures" }
 rustc_errors = { path = "../rustc_errors" }
 rustc_fluent_macro = { path = "../rustc_fluent_macro" }

compiler/rustc_monomorphize/messages.ftl

@@ -1,3 +1,12 @@
+monomorphize_abi_error_disabled_vector_type_call =
+ ABI error: this function call uses a {$required_feature} vector type, which is not enabled in the caller
+ .label = function called here
+ .help = consider enabling it globally (-C target-feature=+{$required_feature}) or locally (#[target_feature(enable="{$required_feature}")])
+monomorphize_abi_error_disabled_vector_type_def =
+ ABI error: this function definition uses a {$required_feature} vector type, which is not enabled
+ .label = function defined here
+ .help = consider enabling it globally (-C target-feature=+{$required_feature}) or locally (#[target_feature(enable="{$required_feature}")])
+
 monomorphize_couldnt_dump_mono_stats =
  unexpected error occurred while dumping monomorphization stats: {$error}
 

compiler/rustc_monomorphize/src/collector.rs

@@ -205,6 +205,7 @@
 //! this is not implemented however: a mono item will be produced
 //! regardless of whether it is actually needed or not.
 
+mod abi_check;
 mod move_check;
 
 use std::path::PathBuf;
@@ -766,6 +767,7 @@ impl<'a, 'tcx> MirVisitor<'tcx> for MirUsedCollector<'a, 'tcx> {
  self.used_mentioned_items.insert(MentionedItem::Fn(callee_ty));
  let callee_ty = self.monomorphize(callee_ty);
  self.check_fn_args_move_size(callee_ty, args, *fn_span, location);
+ abi_check::check_call_site_abi(tcx, callee_ty, *fn_span, self.body.source.instance);
  visit_fn_use(self.tcx, callee_ty, true, source, &mut self.used_items)
  }
  mir::TerminatorKind::Drop { ref place, .. } => {
@@ -1207,6 +1209,9 @@ fn collect_items_of_instance<'tcx>(
  mentioned_items: &mut MonoItems<'tcx>,
  mode: CollectionMode,
 ) {
+ // Check the instance for feature-dependent ABI.
+ abi_check::check_instance_abi(tcx, instance);
+
  let body = tcx.instance_mir(instance.def);
  // Naively, in "used" collection mode, all functions get added to *both* `used_items` and
  // `mentioned_items`. Mentioned items processing will then notice that they have already been

compiler/rustc_monomorphize/src/collector/abi_check.rs

@@ -0,0 +1,108 @@
+use rustc_abi::Abi;
+use rustc_hir::CRATE_HIR_ID;
+use rustc_middle::ty::{self, Instance, InstanceKind, ParamEnv, Ty, TyCtxt};
+use rustc_session::lint::builtin::ABI_ERROR_DISABLED_VECTOR_TYPE;
+use rustc_span::def_id::DefId;
+use rustc_span::{Span, Symbol};
+use rustc_target::abi::call::{FnAbi, PassMode};
+
+use crate::errors::{AbiErrorDisabledVectorTypeCall, AbiErrorDisabledVectorTypeDef};
+
+// Represents the least-constraining feature that is required for vector types up to a certain size
+// to have their "proper" ABI.
+const X86_VECTOR_FEATURES: &'static [(u64, &'static str)] =
+ &[(128, "sse"), (256, "avx"), (512, "avx512f")];
+
+const AARCH64_VECTOR_FEATURES: &'static [(u64, &'static str)] = &[(128, "neon")];
+
+fn do_check_abi<'tcx>(
+ tcx: TyCtxt<'tcx>,
+ abi: &FnAbi<'tcx, Ty<'tcx>>,
+ target_feature_def: DefId,
+ emit_err: impl Fn(&'static str),
+) {
+ let feature_def = if tcx.sess.target.arch == "x86" || tcx.sess.target.arch == "x86_64" {
+ X86_VECTOR_FEATURES
+ } else if tcx.sess.target.arch == "aarch64" {
+ AARCH64_VECTOR_FEATURES
+ } else {
+ // FIXME: add support for non-tier1 architectures
+ return;
+ };
+ let codegen_attrs = tcx.codegen_fn_attrs(target_feature_def);
+ for arg_abi in abi.args.iter().chain(std::iter::once(&abi.ret)) {
+ let size = arg_abi.layout.size;
+ if matches!(arg_abi.layout.abi, Abi::Vector { .. })
+ && !matches!(arg_abi.mode, PassMode::Indirect { .. })
+ {
+ let feature = match feature_def.iter().find(|(bits, _)| size.bits() <= *bits) {
+ Some((_, feature)) => feature,
+ None => panic!("Unknown vector size: {}; arg = {:?}", size.bits(), arg_abi),
+ };
+ let feature_sym = Symbol::intern(feature);
+ if !tcx.sess.unstable_target_features.contains(&feature_sym)
+ && !codegen_attrs.target_features.iter().any(|x| x.name == feature_sym)
+ {
+ emit_err(feature);
+ }
+ }
+ }
+}
+
+/// Checks that the ABI of a given instance of a function does not contain vector-passed arguments
+/// or return values for which the corresponding target feature is not enabled.
+pub(super) fn check_instance_abi<'tcx>(tcx: TyCtxt<'tcx>, instance: Instance<'tcx>) {
+ let param_env = ParamEnv::reveal_all();
+ let Ok(abi) = tcx.fn_abi_of_instance(param_env.and((instance, ty::List::empty()))) else {
+ // An error will be reported during codegen if we cannot determine the ABI of this
+ // function.
+ return;
+ };
+ do_check_abi(tcx, abi, instance.def_id(), |required_feature| {
+ let span = tcx.def_span(instance.def_id());
+ tcx.emit_node_span_lint(
+ ABI_ERROR_DISABLED_VECTOR_TYPE,
+ CRATE_HIR_ID,
+ span,
+ AbiErrorDisabledVectorTypeDef { span, required_feature },
+ );
+ })
+}
+
+/// Checks that a call expression does not try to pass a vector-passed argument which requires a
+/// target feature that the caller does not have, as doing so causes UB because of ABI mismatch.
+pub(super) fn check_call_site_abi<'tcx>(
+ tcx: TyCtxt<'tcx>,
+ ty: Ty<'tcx>,
+ span: Span,
+ caller: InstanceKind<'tcx>,
+) {
+ let param_env = ParamEnv::reveal_all();
+ let callee_abi = match *ty.kind() {
+ ty::FnPtr(..) => tcx.fn_abi_of_fn_ptr(param_env.and((ty.fn_sig(tcx), ty::List::empty()))),
+ ty::FnDef(def_id, args) => {
+ // Intrinsics are handled separately by the compiler.
+ if tcx.intrinsic(def_id).is_some() {
+ return;
+ }
+ let instance = ty::Instance::expect_resolve(tcx, param_env, def_id, args, span);
+ tcx.fn_abi_of_instance(param_env.and((instance, ty::List::empty())))
+ }
+ _ => {
+ panic!("Invalid function call");
+ }
+ };
+
+ let Ok(callee_abi) = callee_abi else {
+ // ABI failed to compute; this will not get through codegen.
+ return;
+ };
+ do_check_abi(tcx, callee_abi, caller.def_id(), |required_feature| {
+ tcx.emit_node_span_lint(
+ ABI_ERROR_DISABLED_VECTOR_TYPE,
+ CRATE_HIR_ID,
+ span,
+ AbiErrorDisabledVectorTypeCall { span, required_feature },
+ );
+ })
+}

compiler/rustc_monomorphize/src/errors.rs

@@ -92,3 +92,21 @@ pub(crate) struct StartNotFound;
 pub(crate) struct UnknownCguCollectionMode<'a> {
  pub mode: &'a str,
 }
+
+#[derive(LintDiagnostic)]
+#[diag(monomorphize_abi_error_disabled_vector_type_def)]
+#[help]
+pub(crate) struct AbiErrorDisabledVectorTypeDef<'a> {
+ #[label]
+ pub span: Span,
+ pub required_feature: &'a str,
+}
+
+#[derive(LintDiagnostic)]
+#[diag(monomorphize_abi_error_disabled_vector_type_call)]
+#[help]
+pub(crate) struct AbiErrorDisabledVectorTypeCall<'a> {
+ #[label]
+ pub span: Span,
+ pub required_feature: &'a str,
+}

library/core/src/primitive_docs.rs

@@ -1752,8 +1752,7 @@ mod prim_ref {}
 ///
 /// For two signatures to be considered *ABI-compatible*, they must use a compatible ABI string,
 /// must take the same number of arguments, the individual argument types and the return types must
-/// be ABI-compatible, and the target feature requirements must be met (see the subsection below for
-/// the last point). The ABI string is declared via `extern "ABI" fn(...) -> ...`; note that
+/// be ABI-compatible. The ABI string is declared via `extern "ABI" fn(...) -> ...`; note that
 /// `fn name(...) -> ...` implicitly uses the `"Rust"` ABI string and `extern fn name(...) -> ...`
 /// implicitly uses the `"C"` ABI string.
 ///
@@ -1821,24 +1820,6 @@ mod prim_ref {}
 /// Behavior since transmuting `None::<NonZero<i32>>` to `NonZero<i32>` violates the non-zero
 /// requirement.
 ///
-/// #### Requirements concerning target features
-///
-/// Under some conditions, the signature used by the caller and the callee can be ABI-incompatible
-/// even if the exact same ABI string and types are being used. As an example, the
-/// `std::arch::x86_64::__m256` type has a different `extern "C"` ABI when the `avx` feature is
-/// enabled vs when it is not enabled.
-///
-/// Therefore, to ensure ABI compatibility when code using different target features is combined
-/// (such as via `#[target_feature]`), we further require that one of the following conditions is
-/// met:
-///
-/// - The function uses the `"Rust"` ABI string (which is the default without `extern`).
-/// - Caller and callee are using the exact same set of target features. For the callee we consider
-/// the features enabled (via `#[target_feature]` and `-C target-feature`/`-C target-cpu`) at the
-/// declaration site; for the caller we consider the features enabled at the call site.
-/// - Neither any argument nor the return value involves a SIMD type (`#[repr(simd)]`) that is not
-/// behind a pointer indirection (i.e., `*mut __m256` is fine, but `(i32, __m256)` is not).
-///
 /// ### Trait implementations
 ///
 /// In this documentation the shorthand `fn(T₁, T₂, …, Tₙ)` is used to represent non-variadic

tests/ui/layout/post-mono-layout-cycle-2.rs

@@ -45,7 +45,6 @@ where
  T: Blah,
 {
  async fn ice(&mut self) {
- //~^ ERROR a cycle occurred during layout computation
  let arr: [(); 0] = [];
  self.t.iter(arr.into_iter()).await;
  }

tests/ui/layout/post-mono-layout-cycle-2.stderr

@@ -12,12 +12,12 @@ LL | Blah::iter(self, iterator).await
  |
  = note: a recursive `async fn` call must introduce indirection such as `Box::pin` to avoid an infinitely sized future
 
-error: a cycle occurred during layout computation
- --> $DIR/post-mono-layout-cycle-2.rs:47:5
+note: the above error was encountered while instantiating `fn main::{closure#0}`
+ --> $DIR/post-mono-layout-cycle-2.rs:16:15
  |
-LL | async fn ice(&mut self) {
- | ^^^^^^^^^^^^^^^^^^^^^^^
+LL |  match fut.as_mut().poll(ctx) {
+ |  ^^^^^^^^^^^^^^^^^^^^^^
 
-error: aborting due to 2 previous errors
+error: aborting due to 1 previous error
 
 For more information about this error, try `rustc --explain E0733`.

tests/ui/layout/post-mono-layout-cycle.rs

@@ -14,7 +14,6 @@ struct Wrapper<T: Trait> {
 }
 
 fn abi<T: Trait>(_: Option<Wrapper<T>>) {}
-//~^ ERROR a cycle occurred during layout computation
 
 fn indirect<T: Trait>() {
  abi::<T>(None);

tests/ui/layout/post-mono-layout-cycle.stderr

@@ -5,12 +5,12 @@ error[E0391]: cycle detected when computing layout of `Wrapper<()>`
  = note: cycle used when computing layout of `core::option::Option<Wrapper<()>>`
  = note: see https://rustc-dev-guide.rust-lang.org/overview.html#queries and https://rustc-dev-guide.rust-lang.org/query.html for more information
 
-error: a cycle occurred during layout computation
- --> $DIR/post-mono-layout-cycle.rs:16:1
+note: the above error was encountered while instantiating `fn indirect::<()>`
+ --> $DIR/post-mono-layout-cycle.rs:23:5
  |
-LL | fn abi<T: Trait>(_: Option<Wrapper<T>>) {}
- | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+LL |  indirect::<()>();
+ |  ^^^^^^^^^^^^^^^^
 
-error: aborting due to 2 previous errors
+error: aborting due to 1 previous error
 
 For more information about this error, try `rustc --explain E0391`.