Unrolled build for #124741

Rollup merge of #124741 - nebulark:patchable-function-entries-pr, r=estebank,workingjubilee

patchable-function-entry: Add unstable compiler flag and attribute

Tracking issue: #123115

Add the -Z patchable-function-entry compiler flag and the #[patchable_function_entry(prefix_nops = m, entry_nops = n)] attribute.
Rebased and adjusted the canditate implementation to match changes in the RFC.

compiler/rustc_codegen_llvm/src/attributes.rs

@@ -2,7 +2,7 @@
 
 use rustc_codegen_ssa::traits::*;
 use rustc_hir::def_id::DefId;
-use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags;
+use rustc_middle::middle::codegen_fn_attrs::{CodegenFnAttrFlags, PatchableFunctionEntry};
 use rustc_middle::ty::{self, TyCtxt};
 use rustc_session::config::{FunctionReturn, OptLevel};
 use rustc_span::symbol::sym;
@@ -53,6 +53,34 @@ fn inline_attr<'ll>(cx: &CodegenCx<'ll, '_>, inline: InlineAttr) -> Option<&'ll
  }
 }
 
+#[inline]
+fn patchable_function_entry_attrs<'ll>(
+ cx: &CodegenCx<'ll, '_>,
+ attr: Option<PatchableFunctionEntry>,
+) -> SmallVec<[&'ll Attribute; 2]> {
+ let mut attrs = SmallVec::new();
+ let patchable_spec = attr.unwrap_or_else(|| {
+ PatchableFunctionEntry::from_config(cx.tcx.sess.opts.unstable_opts.patchable_function_entry)
+ });
+ let entry = patchable_spec.entry();
+ let prefix = patchable_spec.prefix();
+ if entry > 0 {
+ attrs.push(llvm::CreateAttrStringValue(
+ cx.llcx,
+ "patchable-function-entry",
+ &format!("{}", entry),
+ ));
+ }
+ if prefix > 0 {
+ attrs.push(llvm::CreateAttrStringValue(
+ cx.llcx,
+ "patchable-function-prefix",
+ &format!("{}", prefix),
+ ));
+ }
+ attrs
+}
+
 /// Get LLVM sanitize attributes.
 #[inline]
 pub fn sanitize_attrs<'ll>(
@@ -421,6 +449,7 @@ pub fn from_fn_attrs<'ll, 'tcx>(
  llvm::set_alignment(llfn, align);
  }
  to_add.extend(sanitize_attrs(cx, codegen_fn_attrs.no_sanitize));
+ to_add.extend(patchable_function_entry_attrs(cx, codegen_fn_attrs.patchable_function_entry));
 
  // Always annotate functions with the target-cpu they are compiled for.
  // Without this, ThinLTO won't inline Rust functions into Clang generated

compiler/rustc_codegen_ssa/src/codegen_attrs.rs

@@ -1,11 +1,13 @@
 use rustc_ast::{ast, attr, MetaItemKind, NestedMetaItem};
 use rustc_attr::{list_contains_name, InlineAttr, InstructionSetAttr, OptimizeAttr};
-use rustc_errors::{codes::*, struct_span_code_err};
+use rustc_errors::{codes::*, struct_span_code_err, DiagMessage, SubdiagMessage};
 use rustc_hir as hir;
 use rustc_hir::def::DefKind;
 use rustc_hir::def_id::{DefId, LocalDefId, LOCAL_CRATE};
 use rustc_hir::{lang_items, weak_lang_items::WEAK_LANG_ITEMS, LangItem};
-use rustc_middle::middle::codegen_fn_attrs::{CodegenFnAttrFlags, CodegenFnAttrs};
+use rustc_middle::middle::codegen_fn_attrs::{
+ CodegenFnAttrFlags, CodegenFnAttrs, PatchableFunctionEntry,
+};
 use rustc_middle::mir::mono::Linkage;
 use rustc_middle::query::Providers;
 use rustc_middle::ty::{self as ty, TyCtxt};
@@ -447,6 +449,80 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: LocalDefId) -> CodegenFnAttrs {
  None
  };
  }
+ sym::patchable_function_entry => {
+ codegen_fn_attrs.patchable_function_entry = attr.meta_item_list().and_then(|l| {
+ let mut prefix = None;
+ let mut entry = None;
+ for item in l {
+ let Some(meta_item) = item.meta_item() else {
+ tcx.dcx().span_err(item.span(), "expected name value pair");
+ continue;
+ };
+
+ let Some(name_value_lit) = meta_item.name_value_literal() else {
+ tcx.dcx().span_err(item.span(), "expected name value pair");
+ continue;
+ };
+
+ fn emit_error_with_label(
+ tcx: TyCtxt<'_>,
+ span: Span,
+ error: impl Into<DiagMessage>,
+ label: impl Into<SubdiagMessage>,
+ ) {
+ let mut err: rustc_errors::Diag<'_, _> =
+ tcx.dcx().struct_span_err(span, error);
+ err.span_label(span, label);
+ err.emit();
+ }
+
+ let attrib_to_write = match meta_item.name_or_empty() {
+ sym::prefix_nops => &mut prefix,
+ sym::entry_nops => &mut entry,
+ _ => {
+ emit_error_with_label(
+ tcx,
+ item.span(),
+ "unexpected parameter name",
+ format!("expected {} or {}", sym::prefix_nops, sym::entry_nops),
+ );
+ continue;
+ }
+ };
+
+ let rustc_ast::LitKind::Int(val, _) = name_value_lit.kind else {
+ emit_error_with_label(
+ tcx,
+ name_value_lit.span,
+ "invalid literal value",
+ "value must be an integer between `0` and `255`",
+ );
+ continue;
+ };
+
+ let Ok(val) = val.get().try_into() else {
+ emit_error_with_label(
+ tcx,
+ name_value_lit.span,
+ "integer value out of range",
+ "value must be between `0` and `255`",
+ );
+ continue;
+ };
+
+ *attrib_to_write = Some(val);
+ }
+
+ if let (None, None) = (prefix, entry) {
+ tcx.dcx().span_err(attr.span, "must specify at least one parameter");
+ }
+
+ Some(PatchableFunctionEntry::from_prefix_and_entry(
+ prefix.unwrap_or(0),
+ entry.unwrap_or(0),
+ ))
+ })
+ }
  _ => {}
  }
  }

compiler/rustc_feature/src/builtin_attrs.rs

@@ -585,6 +585,13 @@ pub const BUILTIN_ATTRIBUTES: &[BuiltinAttribute] = &[
  EncodeCrossCrate::No, derive_smart_pointer, experimental!(pointee)
  ),
 
+ // RFC 3543
+ // `#[patchable_function_entry(prefix_nops = m, entry_nops = n)]`
+ gated!(
+ patchable_function_entry, Normal, template!(List: "prefix_nops = m, entry_nops = n"), ErrorPreceding,
+ EncodeCrossCrate::Yes, experimental!(patchable_function_entry)
+ ),
+
  // ==========================================================================
  // Internal attributes: Stability, deprecation, and unsafe:
  // ==========================================================================

compiler/rustc_feature/src/unstable.rs

@@ -563,6 +563,8 @@ declare_features! (
  (unstable, offset_of_slice, "CURRENT_RUSTC_VERSION", Some(126151)),
  /// Allows using `#[optimize(X)]`.
  (unstable, optimize_attribute, "1.34.0", Some(54882)),
+ /// Allows specifying nop padding on functions for dynamic patching.
+ (unstable, patchable_function_entry, "CURRENT_RUSTC_VERSION", Some(123115)),
  /// Allows postfix match `expr.match { ... }`
  (unstable, postfix_match, "1.79.0", Some(121618)),
  /// Allows `use<'a, 'b, A, B>` in `impl Trait + use<...>` for precise capture of generic args.

compiler/rustc_interface/src/tests.rs

@@ -8,8 +8,8 @@ use rustc_session::config::{
  ErrorOutputType, ExternEntry, ExternLocation, Externs, FunctionReturn, InliningThreshold,
  Input, InstrumentCoverage, InstrumentXRay, LinkSelfContained, LinkerPluginLto, LocationDetail,
  LtoCli, NextSolverConfig, OomStrategy, Options, OutFileName, OutputType, OutputTypes, PAuthKey,
- PacRet, Passes, Polonius, ProcMacroExecutionStrategy, Strip, SwitchWithOptPath,
- SymbolManglingVersion, WasiExecModel,
+ PacRet, Passes, PatchableFunctionEntry, Polonius, ProcMacroExecutionStrategy, Strip,
+ SwitchWithOptPath, SymbolManglingVersion, WasiExecModel,
 };
 use rustc_session::lint::Level;
 use rustc_session::search_paths::SearchPath;
@@ -813,6 +813,11 @@ fn test_unstable_options_tracking_hash() {
  tracked!(packed_bundled_libs, true);
  tracked!(panic_abort_tests, true);
  tracked!(panic_in_drop, PanicStrategy::Abort);
+ tracked!(
+ patchable_function_entry,
+ PatchableFunctionEntry::from_total_and_prefix_nops(10, 5)
+ .expect("total must be greater than or equal to prefix")
+ );
  tracked!(plt, Some(true));
  tracked!(polonius, Polonius::Legacy);
  tracked!(precise_enum_drop_elaboration, false);

compiler/rustc_middle/src/middle/codegen_fn_attrs.rs

@@ -45,6 +45,32 @@ pub struct CodegenFnAttrs {
  /// The `#[repr(align(...))]` attribute. Indicates the value of which the function should be
  /// aligned to.
  pub alignment: Option<Align>,
+ /// The `#[patchable_function_entry(...)]` attribute. Indicates how many nops should be around
+ /// the function entry.
+ pub patchable_function_entry: Option<PatchableFunctionEntry>,
+}
+
+#[derive(Copy, Clone, Debug, TyEncodable, TyDecodable, HashStable)]
+pub struct PatchableFunctionEntry {
+ /// Nops to prepend to the function
+ prefix: u8,
+ /// Nops after entry, but before body
+ entry: u8,
+}
+
+impl PatchableFunctionEntry {
+ pub fn from_config(config: rustc_session::config::PatchableFunctionEntry) -> Self {
+ Self { prefix: config.prefix(), entry: config.entry() }
+ }
+ pub fn from_prefix_and_entry(prefix: u8, entry: u8) -> Self {
+ Self { prefix, entry }
+ }
+ pub fn prefix(&self) -> u8 {
+ self.prefix
+ }
+ pub fn entry(&self) -> u8 {
+ self.entry
+ }
 }
 
 #[derive(Clone, Copy, PartialEq, Eq, TyEncodable, TyDecodable, HashStable)]
@@ -121,6 +147,7 @@ impl CodegenFnAttrs {
  no_sanitize: SanitizerSet::empty(),
  instruction_set: None,
  alignment: None,
+ patchable_function_entry: None,
  }
  }
 

compiler/rustc_session/src/config.rs

@@ -2965,8 +2965,9 @@ pub(crate) mod dep_tracking {
  CrateType, DebugInfo, DebugInfoCompression, ErrorOutputType, FunctionReturn,
  InliningThreshold, InstrumentCoverage, InstrumentXRay, LinkerPluginLto, LocationDetail,
  LtoCli, NextSolverConfig, OomStrategy, OptLevel, OutFileName, OutputType, OutputTypes,
- Polonius, RemapPathScopeComponents, ResolveDocLinks, SourceFileHashAlgorithm,
- SplitDwarfKind, SwitchWithOptPath, SymbolManglingVersion, WasiExecModel,
+ PatchableFunctionEntry, Polonius, RemapPathScopeComponents, ResolveDocLinks,
+ SourceFileHashAlgorithm, SplitDwarfKind, SwitchWithOptPath, SymbolManglingVersion,
+ WasiExecModel,
  };
  use crate::lint;
  use crate::utils::NativeLib;
@@ -3073,6 +3074,7 @@ pub(crate) mod dep_tracking {
  OomStrategy,
  LanguageIdentifier,
  NextSolverConfig,
+ PatchableFunctionEntry,
  Polonius,
  InliningThreshold,
  FunctionReturn,
@@ -3250,6 +3252,35 @@ impl DumpMonoStatsFormat {
  }
 }
 
+/// `-Z patchable-function-entry` representation - how many nops to put before and after function
+/// entry.
+#[derive(Clone, Copy, PartialEq, Hash, Debug, Default)]
+pub struct PatchableFunctionEntry {
+ /// Nops before the entry
+ prefix: u8,
+ /// Nops after the entry
+ entry: u8,
+}
+
+impl PatchableFunctionEntry {
+ pub fn from_total_and_prefix_nops(
+ total_nops: u8,
+ prefix_nops: u8,
+ ) -> Option<PatchableFunctionEntry> {
+ if total_nops < prefix_nops {
+ None
+ } else {
+ Some(Self { prefix: prefix_nops, entry: total_nops - prefix_nops })
+ }
+ }
+ pub fn prefix(&self) -> u8 {
+ self.prefix
+ }
+ pub fn entry(&self) -> u8 {
+ self.entry
+ }
+}
+
 /// `-Zpolonius` values, enabling the borrow checker polonius analysis, and which version: legacy,
 /// or future prototype.
 #[derive(Clone, Copy, PartialEq, Hash, Debug, Default)]

compiler/rustc_session/src/options.rs

@@ -379,6 +379,7 @@ mod desc {
  pub const parse_passes: &str = "a space-separated list of passes, or `all`";
  pub const parse_panic_strategy: &str = "either `unwind` or `abort`";
  pub const parse_on_broken_pipe: &str = "either `kill`, `error`, or `inherit`";
+ pub const parse_patchable_function_entry: &str = "either two comma separated integers (total_nops,prefix_nops), with prefix_nops <= total_nops, or one integer (total_nops)";
  pub const parse_opt_panic_strategy: &str = parse_panic_strategy;
  pub const parse_oom_strategy: &str = "either `panic` or `abort`";
  pub const parse_relro_level: &str = "one of: `full`, `partial`, or `off`";
@@ -734,6 +735,32 @@ mod parse {
  true
  }
 
+ pub(crate) fn parse_patchable_function_entry(
+ slot: &mut PatchableFunctionEntry,
+ v: Option<&str>,
+ ) -> bool {
+ let mut total_nops = 0;
+ let mut prefix_nops = 0;
+
+ if !parse_number(&mut total_nops, v) {
+ let parts = v.and_then(|v| v.split_once(',')).unzip();
+ if !parse_number(&mut total_nops, parts.0) {
+ return false;
+ }
+ if !parse_number(&mut prefix_nops, parts.1) {
+ return false;
+ }
+ }
+
+ if let Some(pfe) =
+ PatchableFunctionEntry::from_total_and_prefix_nops(total_nops, prefix_nops)
+ {
+ *slot = pfe;
+ return true;
+ }
+ false
+ }
+
  pub(crate) fn parse_oom_strategy(slot: &mut OomStrategy, v: Option<&str>) -> bool {
  match v {
  Some("panic") => *slot = OomStrategy::Panic,
@@ -1859,6 +1886,8 @@ options! {
  "panic strategy for panics in drops"),
  parse_only: bool = (false, parse_bool, [UNTRACKED],
  "parse only; do not compile, assemble, or link (default: no)"),
+ patchable_function_entry: PatchableFunctionEntry = (PatchableFunctionEntry::default(), parse_patchable_function_entry, [TRACKED],
+ "nop padding at function entry"),
  plt: Option<bool> = (None, parse_opt_bool, [TRACKED],
  "whether to use the PLT when calling into shared libraries;
  only has effect for PIC code on systems with ELF binaries

compiler/rustc_span/src/symbol.rs

@@ -768,6 +768,7 @@ symbols! {
  enable,
  encode,
  end,
+ entry_nops,
  enumerate_method,
  env,
  env_CFG_RELEASE: env!("CFG_RELEASE"),
@@ -1383,6 +1384,7 @@ symbols! {
  passes,
  pat,
  pat_param,
+ patchable_function_entry,
  path,
  pattern_complexity,
  pattern_parentheses,
@@ -1421,6 +1423,7 @@ symbols! {
  prefetch_read_instruction,
  prefetch_write_data,
  prefetch_write_instruction,
+ prefix_nops,
  preg,
  prelude,
  prelude_import,

src/doc/unstable-book/src/compiler-flags/patchable-function-entry.md

@@ -0,0 +1,24 @@
+# `patchable-function-entry`
+
+--------------------
+
+The `-Z patchable-function-entry=total_nops,prefix_nops` or `-Z patchable-function-entry=total_nops`
+compiler flag enables nop padding of function entries with 'total_nops' nops, with
+an offset for the entry of the function at 'prefix_nops' nops. In the second form,
+'prefix_nops' defaults to 0.
+
+As an illustrative example, `-Z patchable-function-entry=3,2` would produce:
+
+```text
+nop
+nop
+function_label:
+nop
+//Actual function code begins here
+```
+
+This flag is used for hotpatching, especially in the Linux kernel. The flag
+arguments are modeled after the `-fpatchable-function-entry` flag as defined
+for both [Clang](https://clang.llvm.org/docs/ClangCommandLineReference.html#cmdoption-clang-fpatchable-function-entry)
+and [gcc](https://gcc.gnu.org/onlinedocs/gcc/Instrumentation-Options.html#index-fpatchable-function-entry)
+and is intended to provide the same effect.