Skip to content

Commit 10f4c23

Browse files
DiamondLovesYouDiamondLovesYou
committed
Driver provided "intrinsics".
Allows drivers to define functions which are only expanded by the plugin after all type checking, inference, etc etc, ie only once during mono item collection. The form of the interface with the plugins is slightly different than what was proposed in rust-lang#51623. Additionally, signature checking is added.
1 parent 724956f commit 10f4c23

File tree

23 files changed

+481
-27
lines changed

23 files changed

+481
-27
lines changed

Diff for: ‎compiler/rustc_codegen_ssa/src/mir/block.rs

+8-2
Original file line numberDiff line numberDiff line change
@@ -624,9 +624,14 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
624624
let arg_count = fn_abi.args.len() + fn_abi.ret.is_indirect() as usize;
625625
let mut llargs = Vec::with_capacity(arg_count);
626626

627+
// Custom intrinsics are treated as-if they were normal functions here.
628+
let is_custom_intrinsic = intrinsic.and_then(|_| instance )
629+
.map(|instance| bx.tcx().custom_intrinsic_mir(instance).is_some() )
630+
.unwrap_or_default();
631+
627632
// Prepare the return value destination
628633
let ret_dest = if let Some((dest, _)) = *destination {
629-
let is_intrinsic = intrinsic.is_some();
634+
let is_intrinsic = intrinsic.is_some() && !is_custom_intrinsic;
630635
self.make_return_dest(&mut bx, dest, &fn_abi.ret, &mut llargs, is_intrinsic)
631636
} else {
632637
ReturnDest::Nothing
@@ -647,7 +652,8 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
647652
return;
648653
}
649654

650-
if intrinsic.is_some() && intrinsic != Some(sym::drop_in_place) {
655+
if intrinsic.is_some() && intrinsic != Some(sym::drop_in_place) &&
656+
!is_custom_intrinsic {
651657
let intrinsic = intrinsic.unwrap();
652658
let dest = match ret_dest {
653659
_ if fn_abi.ret.is_indirect() => llargs[0],

Diff for: ‎compiler/rustc_codegen_ssa/src/mir/mod.rs

+1-1
Original file line numberDiff line numberDiff line change
@@ -140,7 +140,7 @@ pub fn codegen_mir<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
140140

141141
let llfn = cx.get_fn(instance);
142142

143-
let mir = cx.tcx().instance_mir(instance.def);
143+
let mir = cx.tcx().instance_mir(instance);
144144

145145
let fn_abi = FnAbi::of_instance(cx, instance, &[]);
146146
debug!("fn_abi: {:?}", fn_abi);

Diff for: ‎compiler/rustc_data_structures/src/sync.rs

+53
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ use crate::owning_ref::{Erased, OwningRef};
2121
use std::collections::HashMap;
2222
use std::hash::{BuildHasher, Hash};
2323
use std::ops::{Deref, DerefMut};
24+
use std::sync::{Arc, atomic::AtomicPtr, atomic, };
2425

2526
pub use std::sync::atomic::Ordering;
2627
pub use std::sync::atomic::Ordering::SeqCst;
@@ -656,3 +657,55 @@ impl<T> DerefMut for OneThread<T> {
656657
&mut self.inner
657658
}
658659
}
660+
661+
/// Provides atomic mutability by replacing the value inside this `ArcCell`.
662+
/// Similar to the `crossbeam` structure of the same name.
663+
#[derive(Debug)]
664+
pub struct ArcCell<T>(AtomicPtr<T>);
665+
impl<T> ArcCell<T> {
666+
pub fn new(v: Arc<T>) -> Self {
667+
ArcCell(AtomicPtr::new(Arc::into_raw(v) as *mut _))
668+
}
669+
pub fn get(&self) -> Arc<T> {
670+
let ptr = self.0.load(atomic::Ordering::Acquire);
671+
let arc = unsafe { Arc::from_raw(ptr as *const T) };
672+
let ret = arc.clone();
673+
// don't drop our copy:
674+
::std::mem::forget(arc);
675+
ret
676+
}
677+
/// Update the value, returning the previous value.
678+
pub fn set(&self, v: Arc<T>) -> Arc<T> {
679+
let new = Arc::into_raw(v) as *mut _;
680+
let mut expected = self.0.load(atomic::Ordering::Acquire);
681+
loop {
682+
match self.0.compare_exchange_weak(expected, new,
683+
atomic::Ordering::SeqCst,
684+
atomic::Ordering::Acquire) {
685+
Ok(old) => {
686+
return unsafe { Arc::from_raw(old as *const T) };
687+
},
688+
Err(v) => {
689+
expected = v;
690+
},
691+
}
692+
}
693+
}
694+
}
695+
impl<T> Drop for ArcCell<T> {
696+
fn drop(&mut self) {
697+
let ptr = self.0.load(atomic::Ordering::Acquire);
698+
// drop our copy of the arc:
699+
unsafe { Arc::from_raw(ptr as *const _) };
700+
}
701+
}
702+
impl<T> Clone for ArcCell<T> {
703+
fn clone(&self) -> Self {
704+
ArcCell::new(self.get())
705+
}
706+
}
707+
impl<T> From<Arc<T>> for ArcCell<T> {
708+
fn from(v: Arc<T>) -> Self {
709+
Self::new(v)
710+
}
711+
}

Diff for: ‎compiler/rustc_middle/src/mir/mod.rs

+31-1
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,7 @@ mod type_foldable;
5353
pub mod visit;
5454

5555
/// Types for locals
56-
type LocalDecls<'tcx> = IndexVec<Local, LocalDecl<'tcx>>;
56+
pub type LocalDecls<'tcx> = IndexVec<Local, LocalDecl<'tcx>>;
5757

5858
pub trait HasLocalDecls<'tcx> {
5959
fn local_decls(&self) -> &LocalDecls<'tcx>;
@@ -2615,3 +2615,33 @@ impl Location {
26152615
}
26162616
}
26172617
}
2618+
2619+
pub trait CustomIntrinsicMirGen: Sync + Send {
2620+
/// Codegen a plugin-defined intrinsic. This is intended to be used to
2621+
/// "return" values based on the monomorphized and erased types of the
2622+
/// function call. Codegen will codegen the `extra_stmts` and then insert
2623+
/// an unconditional branch to the exit block.
2624+
///
2625+
/// Consider this to be highly unstable; it will likely change without
2626+
/// warning. There is also no spec for this, it is 100% implementation
2627+
/// defined, and may not be implemented at all for some codegen backends.
2628+
///
2629+
/// If the codegen backend is multithreaded, this will be called from
2630+
/// any number of threads, hence `Sync + Send`.
2631+
///
2632+
/// YOU ARE RESPONSIBLE FOR THE SAFETY OF THE EXTRA STATEMENTS.
2633+
/// You have been warned. Good luck, have fun.
2634+
fn mirgen_simple_intrinsic<'tcx>(&self,
2635+
tcx: TyCtxt<'tcx>,
2636+
instance: ty::Instance<'tcx>,
2637+
mir: &mut Body<'tcx>);
2638+
2639+
/// The following are used for typeck-ing:
2640+
2641+
/// The number of generic parameters expected.
2642+
fn generic_parameter_count<'tcx>(&self, tcx: TyCtxt<'tcx>) -> usize;
2643+
/// The types of the input args.
2644+
fn inputs<'tcx>(&self, tcx: TyCtxt<'tcx>) -> &'tcx List<Ty<'tcx>>;
2645+
/// The return type.
2646+
fn output<'tcx>(&self, tcx: TyCtxt<'tcx>) -> Ty<'tcx>;
2647+
}

Diff for: ‎compiler/rustc_middle/src/query/mod.rs

+15
Original file line numberDiff line numberDiff line change
@@ -277,6 +277,21 @@ rustc_queries! {
277277
}
278278
}
279279

280+
/// If defined by the driver, returns the extra mir statements to codegen,
281+
/// else returns `None`.
282+
query custom_intrinsic_mirgen(key: DefId) -> Option<Lrc<dyn mir::CustomIntrinsicMirGen>> {
283+
anon
284+
no_hash
285+
286+
desc { |tcx| "asking for the custom MIR generator of `{}`", tcx.def_path_str(key) }
287+
}
288+
/// The monomorphized MIR for a custom intrinsic instance.
289+
query custom_intrinsic_mir(inst: ty::Instance<'tcx>) -> Option<&'tcx mir::Body<'tcx>> {
290+
anon
291+
292+
desc { |tcx| "asking for the custom MIR of `{}`", tcx.def_path_str(inst.def_id()) }
293+
}
294+
280295
/// Returns coverage summary info for a function, after executing the `InstrumentCoverage`
281296
/// MIR pass (assuming the -Zinstrument-coverage option is enabled).
282297
query coverageinfo(key: DefId) -> mir::CoverageInfo {

Diff for: ‎compiler/rustc_middle/src/ty/mod.rs

+10-4
Original file line numberDiff line numberDiff line change
@@ -2968,23 +2968,29 @@ impl<'tcx> TyCtxt<'tcx> {
29682968
}
29692969

29702970
/// Returns the possibly-auto-generated MIR of a `(DefId, Subst)` pair.
2971-
pub fn instance_mir(self, instance: ty::InstanceDef<'tcx>) -> &'tcx Body<'tcx> {
2972-
match instance {
2971+
pub fn instance_mir(self, instance: ty::Instance<'tcx>) -> &'tcx Body<'tcx> {
2972+
match instance.def {
29732973
ty::InstanceDef::Item(def) => {
29742974
if let Some((did, param_did)) = def.as_const_arg() {
29752975
self.optimized_mir_of_const_arg((did, param_did))
29762976
} else {
29772977
self.optimized_mir(def.did)
29782978
}
29792979
}
2980+
ty::InstanceDef::Intrinsic(..) => {
2981+
if let Some(mir) = self.custom_intrinsic_mir(instance) {
2982+
mir
2983+
} else {
2984+
self.mir_shims(instance.def)
2985+
}
2986+
}
29802987
ty::InstanceDef::VtableShim(..)
29812988
| ty::InstanceDef::ReifyShim(..)
2982-
| ty::InstanceDef::Intrinsic(..)
29832989
| ty::InstanceDef::FnPtrShim(..)
29842990
| ty::InstanceDef::Virtual(..)
29852991
| ty::InstanceDef::ClosureOnceShim { .. }
29862992
| ty::InstanceDef::DropGlue(..)
2987-
| ty::InstanceDef::CloneShim(..) => self.mir_shims(instance),
2993+
| ty::InstanceDef::CloneShim(..) => self.mir_shims(instance.def),
29882994
}
29892995
}
29902996

Diff for: ‎compiler/rustc_mir/src/const_eval/eval_queries.rs

+1-1
Original file line numberDiff line numberDiff line change
@@ -317,7 +317,7 @@ pub fn const_eval_raw_provider<'tcx>(
317317
MemoryExtra { can_access_statics: is_static },
318318
);
319319

320-
let res = ecx.load_mir(cid.instance.def, cid.promoted);
320+
let res = ecx.load_mir(cid.instance, cid.promoted);
321321
res.and_then(|body| eval_body_using_ecx(&mut ecx, cid, &body))
322322
.map(|place| RawConst { alloc_id: place.ptr.assert_ptr().alloc_id, ty: place.layout.ty })
323323
.map_err(|error| {

Diff for: ‎compiler/rustc_mir/src/const_eval/machine.rs

+1-1
Original file line numberDiff line numberDiff line change
@@ -211,7 +211,7 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for CompileTimeInterpreter<'mir,
211211
}
212212
}
213213
// This is a const fn. Call it.
214-
Ok(Some(match ecx.load_mir(instance.def, None) {
214+
Ok(Some(match ecx.load_mir(instance, None) {
215215
Ok(body) => body,
216216
Err(err) => {
217217
if let err_unsup!(NoMirFor(did)) = err.kind {

Diff for: ‎compiler/rustc_mir/src/interpret/eval_context.rs

+3-3
Original file line numberDiff line numberDiff line change
@@ -424,11 +424,11 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
424424

425425
pub fn load_mir(
426426
&self,
427-
instance: ty::InstanceDef<'tcx>,
427+
instance: ty::Instance<'tcx>,
428428
promoted: Option<mir::Promoted>,
429429
) -> InterpResult<'tcx, &'tcx mir::Body<'tcx>> {
430430
// do not continue if typeck errors occurred (can only occur in local crate)
431-
let def = instance.with_opt_param();
431+
let def = instance.def.with_opt_param();
432432
if let Some(def) = def.as_local() {
433433
if self.tcx.has_typeck_results(def.did) {
434434
if let Some(error_reported) = self.tcx.typeck_opt_const_arg(def).tainted_by_errors {
@@ -440,7 +440,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
440440
if let Some(promoted) = promoted {
441441
return Ok(&self.tcx.promoted_mir_of_opt_const_arg(def)[promoted]);
442442
}
443-
match instance {
443+
match instance.def {
444444
ty::InstanceDef::Item(def) => {
445445
if self.tcx.is_mir_available(def.did) {
446446
if let Some((did, param_did)) = def.as_const_arg() {

Diff for: ‎compiler/rustc_mir/src/interpret/terminator.rs

+3
Original file line numberDiff line numberDiff line change
@@ -218,6 +218,8 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
218218
}
219219
};
220220

221+
let custom = self.tcx.custom_intrinsic_mir(instance);
222+
221223
// ABI check
222224
{
223225
let callee_abi = {
@@ -296,6 +298,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
296298
// The Rust ABI is special: ZST get skipped.
297299
let rust_abi = match caller_abi {
298300
Abi::Rust | Abi::RustCall => true,
301+
Abi::RustIntrinsic if custom.is_some() => true,
299302
_ => false,
300303
};
301304
// We have two iterators: Where the arguments come from,

Diff for: ‎compiler/rustc_mir/src/lib.rs

+1
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@ pub fn provide(providers: &mut Providers) {
4949
const_eval::provide(providers);
5050
shim::provide(providers);
5151
transform::provide(providers);
52+
monomorphize::provide(providers);
5253
monomorphize::partitioning::provide(providers);
5354
monomorphize::polymorphize::provide(providers);
5455
providers.const_eval_validated = const_eval::const_eval_validated_provider;

Diff for: ‎compiler/rustc_mir/src/monomorphize/collector.rs

+5-1
Original file line numberDiff line numberDiff line change
@@ -744,6 +744,10 @@ fn visit_instance_use<'tcx>(
744744
if !is_direct_call {
745745
bug!("{:?} being reified", instance);
746746
}
747+
748+
if let Some(_mir) = tcx.custom_intrinsic_mir(instance) {
749+
output.push(create_fn_mono_item(tcx, instance, source));
750+
}
747751
}
748752
ty::InstanceDef::DropGlue(_, None) => {
749753
// Don't need to emit noop drop glue if we are calling directly.
@@ -1212,7 +1216,7 @@ fn collect_neighbours<'tcx>(
12121216
output: &mut Vec<Spanned<MonoItem<'tcx>>>,
12131217
) {
12141218
debug!("collect_neighbours: {:?}", instance.def_id());
1215-
let body = tcx.instance_mir(instance.def);
1219+
let body = tcx.instance_mir(instance);
12161220

12171221
MirNeighborCollector { tcx, body: &body, output, instance }.visit_body(&body);
12181222
}

Diff for: ‎compiler/rustc_mir/src/monomorphize/mod.rs

+68-2
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,79 @@
1-
use rustc_middle::traits;
1+
use rustc_index::vec::IndexVec;
2+
use rustc_middle::{traits, mir};
23
use rustc_middle::ty::adjustment::CustomCoerceUnsized;
3-
use rustc_middle::ty::{self, Ty, TyCtxt};
4+
use rustc_middle::ty::{self, Ty, TyCtxt, Instance};
5+
use rustc_middle::ty::query::Providers;
6+
use rustc_span::DUMMY_SP;
47

58
use rustc_hir::lang_items::LangItem;
69

710
pub mod collector;
811
pub mod partitioning;
912
pub mod polymorphize;
1013

14+
pub fn provide(providers: &mut Providers) {
15+
providers.custom_intrinsic_mirgen = |_, _| { None };
16+
providers.custom_intrinsic_mir = custom_intrinsic_mir;
17+
}
18+
19+
fn custom_intrinsic_mir<'tcx>(tcx: TyCtxt<'tcx>,
20+
instance: Instance<'tcx>)
21+
-> Option<&'tcx mir::Body<'tcx>>
22+
{
23+
let mirgen = tcx.custom_intrinsic_mirgen(instance.def_id())?;
24+
25+
let ty = instance.ty(tcx, ty::ParamEnv::reveal_all());
26+
let sig = ty.fn_sig(tcx);
27+
let sig = tcx.normalize_erasing_late_bound_regions(
28+
ty::ParamEnv::reveal_all(),
29+
&sig,
30+
);
31+
32+
// no var arg calls, so we can skip monomorphizing extra arguments.
33+
assert!(!sig.c_variadic);
34+
35+
let source_scope_local_data = mir::ClearCrossCrate::Clear;
36+
let source_scope = mir::SourceScopeData {
37+
span: DUMMY_SP,
38+
parent_scope: None,
39+
local_data: source_scope_local_data,
40+
};
41+
let source_info = mir::SourceInfo {
42+
span: DUMMY_SP,
43+
scope: mir::OUTERMOST_SOURCE_SCOPE,
44+
};
45+
46+
let mut source_scopes = IndexVec::new();
47+
source_scopes.push(source_scope.clone());
48+
49+
let ret_decl = mir::LocalDecl::new(sig.output(), DUMMY_SP);
50+
let mut local_decls = IndexVec::from_elem_n(ret_decl, 1);
51+
for &arg in sig.inputs().iter() {
52+
local_decls.push(mir::LocalDecl {
53+
mutability: mir::Mutability::Mut,
54+
local_info: None,
55+
ty: arg,
56+
source_info,
57+
internal: false,
58+
user_ty: None,
59+
is_block_tail: None,
60+
});
61+
}
62+
63+
let mut gen = mir::Body::new(IndexVec::new(),
64+
source_scopes,
65+
local_decls,
66+
Default::default(),
67+
sig.inputs().len(),
68+
Vec::new(),
69+
source_scope.span,
70+
None);
71+
72+
mirgen.mirgen_simple_intrinsic(tcx, instance, &mut gen);
73+
74+
Some(tcx.arena.alloc(gen))
75+
}
76+
1177
pub fn custom_coerce_unsize_info<'tcx>(
1278
tcx: TyCtxt<'tcx>,
1379
source_ty: Ty<'tcx>,

Diff for: ‎compiler/rustc_symbol_mangling/src/lib.rs

+4
Original file line numberDiff line numberDiff line change
@@ -178,6 +178,10 @@ fn compute_symbol_name(
178178
Node::ForeignItem(_) => true,
179179
_ => false,
180180
}
181+
} else if let ty::InstanceDef::Intrinsic(_) = instance.def {
182+
// custom intrinsics should never be foreign, otherwise
183+
// generic parameters will cause duplicate symbols names.
184+
tcx.custom_intrinsic_mir(instance).is_none()
181185
} else {
182186
tcx.is_foreign_item(def_id)
183187
};

Diff for: ‎compiler/rustc_ty/src/ty.rs

+4-1
Original file line numberDiff line numberDiff line change
@@ -302,7 +302,10 @@ fn instance_def_size_estimate<'tcx>(
302302

303303
match instance_def {
304304
InstanceDef::Item(..) | InstanceDef::DropGlue(..) => {
305-
let mir = tcx.instance_mir(instance_def);
305+
let mir = tcx.instance_mir(ty::Instance {
306+
def: instance_def,
307+
substs: tcx.intern_substs(&[]),
308+
});
306309
mir.basic_blocks().iter().map(|bb| bb.statements.len()).sum()
307310
}
308311
// Estimate the size of other compiler-generated shims to be 1.

0 commit comments

Comments
 (0)