From 29a4ec0d434dba190dfe9e4d46f16e4c026dd6cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?John=20K=C3=A5re=20Alsaker?= Date: Thu, 15 Mar 2018 10:03:36 +0100 Subject: [PATCH 01/12] Make queries thread safe. Remove the query stack and make queries point to their parents instead. --- src/librustc/ty/context.rs | 182 +++++++++---- src/librustc/ty/maps/job.rs | 91 +++++++ src/librustc/ty/maps/mod.rs | 10 +- src/librustc/ty/maps/on_disk_cache.rs | 9 + src/librustc/ty/maps/plumbing.rs | 351 +++++++++++++++++--------- src/librustc_errors/lib.rs | 25 +- 6 files changed, 486 insertions(+), 182 deletions(-) create mode 100644 src/librustc/ty/maps/job.rs diff --git a/src/librustc/ty/context.rs b/src/librustc/ty/context.rs index 966c96e594fc4..adb0ecd39846a 100644 --- a/src/librustc/ty/context.rs +++ b/src/librustc/ty/context.rs @@ -1222,7 +1222,7 @@ impl<'a, 'gcx, 'tcx> TyCtxt<'a, 'gcx, 'tcx> { Lrc::new(StableVec::new(v))); } - tls::enter_global(GlobalCtxt { + let gcx = &GlobalCtxt { sess: s, cstore, global_arenas: &arenas.global, @@ -1263,7 +1263,9 @@ impl<'a, 'gcx, 'tcx> TyCtxt<'a, 'gcx, 'tcx> { all_traits: RefCell::new(None), tx_to_llvm_workers: tx, output_filenames: Arc::new(output_filenames.clone()), - }, f) + }; + + tls::enter_global(gcx, f) } pub fn consider_optimizing String>(&self, msg: T) -> bool { @@ -1487,11 +1489,25 @@ impl<'a, 'tcx> TyCtxt<'a, 'tcx, 'tcx> { impl<'gcx: 'tcx, 'tcx> GlobalCtxt<'gcx> { /// Call the closure with a local `TyCtxt` using the given arena. - pub fn enter_local(&self, arena: &'tcx DroplessArena, f: F) -> R + pub fn enter_local(&self, + arena: &'tcx DroplessArena, + f: F) -> R where F: for<'a> FnOnce(TyCtxt<'a, 'gcx, 'tcx>) -> R { let interners = CtxtInterners::new(arena); - tls::enter(self, &interners, f) + let tcx = TyCtxt { + gcx: self, + interners: &interners, + }; + ty::tls::with_related_context(tcx.global_tcx(), |icx| { + let new_icx = ty::tls::ImplicitCtxt { + tcx, + query: icx.query.clone(), + }; + ty::tls::enter_context(&new_icx, |new_icx| { + f(new_icx.tcx) + }) + }) } } @@ -1638,21 +1654,34 @@ impl<'a, 'tcx> Lift<'tcx> for &'a Slice> { } pub mod tls { - use super::{CtxtInterners, GlobalCtxt, TyCtxt}; + use super::{GlobalCtxt, TyCtxt}; use std::cell::Cell; use std::fmt; + use std::mem; use syntax_pos; + use ty::maps; + use errors::{Diagnostic, TRACK_DIAGNOSTICS}; + use rustc_data_structures::OnDrop; + use rustc_data_structures::sync::Lrc; + + #[derive(Clone)] + pub struct ImplicitCtxt<'a, 'gcx: 'a+'tcx, 'tcx: 'a> { + pub tcx: TyCtxt<'a, 'gcx, 'tcx>, + pub query: Option>>, + } + + thread_local!(static TLV: Cell = Cell::new(0)); - /// Marker types used for the scoped TLS slot. - /// The type context cannot be used directly because the scoped TLS - /// in libstd doesn't allow types generic over lifetimes. - enum ThreadLocalGlobalCtxt {} - enum ThreadLocalInterners {} + fn set_tlv R, R>(value: usize, f: F) -> R { + let old = get_tlv(); + let _reset = OnDrop(move || TLV.with(|tlv| tlv.set(old))); + TLV.with(|tlv| tlv.set(value)); + f() + } - thread_local! { - static TLS_TCX: Cell> = Cell::new(None) + fn get_tlv() -> usize { + TLV.with(|tlv| tlv.get()) } fn span_debug(span: syntax_pos::Span, f: &mut fmt::Formatter) -> fmt::Result { @@ -1661,59 +1690,120 @@ pub mod tls { }) } - pub fn enter_global<'gcx, F, R>(gcx: GlobalCtxt<'gcx>, f: F) -> R - where F: for<'a> FnOnce(TyCtxt<'a, 'gcx, 'gcx>) -> R + fn track_diagnostic(diagnostic: &Diagnostic) { + with_context(|context| { + if let Some(ref query) = context.query { + query.diagnostics.lock().push(diagnostic.clone()); + } + }) + } + + pub fn with_thread_locals(f: F) -> R + where F: FnOnce() -> R { syntax_pos::SPAN_DEBUG.with(|span_dbg| { let original_span_debug = span_dbg.get(); span_dbg.set(span_debug); - let result = enter(&gcx, &gcx.global_interners, f); - span_dbg.set(original_span_debug); - result + + let _on_drop = OnDrop(move || { + span_dbg.set(original_span_debug); + }); + + TRACK_DIAGNOSTICS.with(|current| { + let original = current.get(); + current.set(track_diagnostic); + + let _on_drop = OnDrop(move || { + current.set(original); + }); + + f() + }) }) } - pub fn enter<'a, 'gcx: 'tcx, 'tcx, F, R>(gcx: &'a GlobalCtxt<'gcx>, - interners: &'a CtxtInterners<'tcx>, - f: F) -> R - where F: FnOnce(TyCtxt<'a, 'gcx, 'tcx>) -> R + pub fn enter_global<'gcx, F, R>(gcx: &GlobalCtxt<'gcx>, f: F) -> R + where F: for<'a> FnOnce(TyCtxt<'a, 'gcx, 'gcx>) -> R { - let gcx_ptr = gcx as *const _ as *const ThreadLocalGlobalCtxt; - let interners_ptr = interners as *const _ as *const ThreadLocalInterners; - TLS_TCX.with(|tls| { - let prev = tls.get(); - tls.set(Some((gcx_ptr, interners_ptr))); - let ret = f(TyCtxt { + with_thread_locals(|| { + let tcx = TyCtxt { gcx, - interners, - }); - tls.set(prev); - ret + interners: &gcx.global_interners, + }; + let icx = ImplicitCtxt { + tcx, + query: None, + }; + enter_context(&icx, |_| { + f(tcx) + }) + }) + } + + pub fn enter_context<'a, 'gcx: 'tcx, 'tcx, F, R>(context: &ImplicitCtxt<'a, 'gcx, 'tcx>, + f: F) -> R + where F: FnOnce(&ImplicitCtxt<'a, 'gcx, 'tcx>) -> R + { + set_tlv(context as *const _ as usize, || { + f(&context) + }) + } + + pub fn with_context_opt(f: F) -> R + where F: for<'a, 'gcx, 'tcx> FnOnce(Option<&ImplicitCtxt<'a, 'gcx, 'tcx>>) -> R + { + let context = get_tlv(); + if context == 0 { + f(None) + } else { + unsafe { f(Some(&*(context as *const ImplicitCtxt))) } + } + } + + pub fn with_fully_related_context<'a, 'gcx, 'tcx, F, R>(tcx: TyCtxt<'a, 'gcx, 'tcx>, f: F) -> R + where F: for<'b> FnOnce(&ImplicitCtxt<'b, 'gcx, 'tcx>) -> R + { + with_context(|context| { + unsafe { + let gcx = tcx.gcx as *const _ as usize; + let interners = tcx.interners as *const _ as usize; + assert!(context.tcx.gcx as *const _ as usize == gcx); + assert!(context.tcx.interners as *const _ as usize == interners); + let context: &ImplicitCtxt = mem::transmute(context); + f(context) + } }) } + pub fn with_related_context<'a, 'gcx, 'tcx1, F, R>(tcx: TyCtxt<'a, 'gcx, 'tcx1>, f: F) -> R + where F: for<'b, 'tcx2> FnOnce(&ImplicitCtxt<'b, 'gcx, 'tcx2>) -> R + { + with_context(|context| { + unsafe { + let gcx = tcx.gcx as *const _ as usize; + assert!(context.tcx.gcx as *const _ as usize == gcx); + let context: &ImplicitCtxt = mem::transmute(context); + f(context) + } + }) + } + + pub fn with_context(f: F) -> R + where F: for<'a, 'gcx, 'tcx> FnOnce(&ImplicitCtxt<'a, 'gcx, 'tcx>) -> R + { + with_context_opt(|opt_context| f(opt_context.expect("no ImplicitCtxt stored in tls"))) + } + pub fn with(f: F) -> R where F: for<'a, 'gcx, 'tcx> FnOnce(TyCtxt<'a, 'gcx, 'tcx>) -> R { - TLS_TCX.with(|tcx| { - let (gcx, interners) = tcx.get().unwrap(); - let gcx = unsafe { &*(gcx as *const GlobalCtxt) }; - let interners = unsafe { &*(interners as *const CtxtInterners) }; - f(TyCtxt { - gcx, - interners, - }) - }) + with_context(|context| f(context.tcx)) } pub fn with_opt(f: F) -> R where F: for<'a, 'gcx, 'tcx> FnOnce(Option>) -> R { - if TLS_TCX.with(|tcx| tcx.get().is_some()) { - with(|v| f(Some(v))) - } else { - f(None) - } + with_context_opt(|opt_context| f(opt_context.map(|context| context.tcx))) } } diff --git a/src/librustc/ty/maps/job.rs b/src/librustc/ty/maps/job.rs new file mode 100644 index 0000000000000..2ed993bd975e5 --- /dev/null +++ b/src/librustc/ty/maps/job.rs @@ -0,0 +1,91 @@ +// Copyright 2017 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +#![allow(warnings)] + +use std::mem; +use std::sync::atomic::AtomicBool; +use std::sync::atomic::Ordering; +use rustc_data_structures::sync::{Lock, LockGuard, Lrc}; +use syntax_pos::Span; +use ty::tls; +use ty::maps::Query; +use ty::maps::plumbing::CycleError; +use ty::context::TyCtxt; +use errors::Diagnostic; +use std::process; +use std::fmt; +use std::sync::{Arc, Mutex}; +use std::collections::HashSet; + +pub struct PoisonedJob; + +#[derive(Clone, Debug)] +pub struct StackEntry<'tcx> { + pub span: Span, + pub query: Query<'tcx>, +} + +pub struct QueryJob<'tcx> { + pub entry: StackEntry<'tcx>, + pub parent: Option>>, + pub track_diagnostics: bool, + pub diagnostics: Lock>, +} + +impl<'tcx> QueryJob<'tcx> { + pub fn new( + entry: StackEntry<'tcx>, + track_diagnostics: bool, + parent: Option>>, + ) -> Self { + QueryJob { + track_diagnostics, + diagnostics: Lock::new(Vec::new()), + entry, + parent, + } + } + + pub(super) fn await<'lcx>( + &self, + tcx: TyCtxt<'_, 'tcx, 'lcx>, + span: Span, + ) -> Result<(), CycleError<'tcx>> { + // The query is already executing, so this must be a cycle for single threaded rustc, + // so we find the cycle and return it + + let mut current_job = tls::with_related_context(tcx, |icx| icx.query.clone()); + let mut cycle = Vec::new(); + + while let Some(job) = current_job { + cycle.insert(0, job.entry.clone()); + + if &*job as *const _ == self as *const _ { + break; + } + + current_job = job.parent.clone(); + } + + Err(CycleError { span, cycle }) + } + + pub fn signal_complete(&self) { + // Signals to waiters that the query is complete. + // This is a no-op for single threaded rustc + } +} + +pub(super) enum QueryResult<'tcx, T> { + Started(Lrc>), + Complete(T), + Poisoned, +} diff --git a/src/librustc/ty/maps/mod.rs b/src/librustc/ty/maps/mod.rs index c16ad0d8ca140..6dd31baa5931b 100644 --- a/src/librustc/ty/maps/mod.rs +++ b/src/librustc/ty/maps/mod.rs @@ -14,7 +14,7 @@ use hir::def_id::{CrateNum, DefId, DefIndex}; use hir::def::{Def, Export}; use hir::{self, TraitCandidate, ItemLocalId, TransFnAttrs}; use hir::svh::Svh; -use infer::canonical::{Canonical, QueryResult}; +use infer::canonical::{self, Canonical}; use lint; use middle::borrowck::BorrowCheckResult; use middle::cstore::{ExternCrate, LinkagePreference, NativeLibrary, @@ -66,6 +66,10 @@ mod plumbing; use self::plumbing::*; pub use self::plumbing::force_from_dep_node; +mod job; +pub use self::job::{QueryJob, StackEntry, PoisonedJob}; +use self::job::QueryResult; + mod keys; pub use self::keys::Key; @@ -399,7 +403,7 @@ define_maps! { <'tcx> [] fn normalize_projection_ty: NormalizeProjectionTy( CanonicalProjectionGoal<'tcx> ) -> Result< - Lrc>>>, + Lrc>>>, NoSolution, >, @@ -412,7 +416,7 @@ define_maps! { <'tcx> [] fn dropck_outlives: DropckOutlives( CanonicalTyGoal<'tcx> ) -> Result< - Lrc>>>, + Lrc>>>, NoSolution, >, diff --git a/src/librustc/ty/maps/on_disk_cache.rs b/src/librustc/ty/maps/on_disk_cache.rs index c103d6e015aa4..4d78703613e94 100644 --- a/src/librustc/ty/maps/on_disk_cache.rs +++ b/src/librustc/ty/maps/on_disk_cache.rs @@ -30,6 +30,7 @@ use syntax::codemap::{CodeMap, StableFilemapId}; use syntax_pos::{BytePos, Span, DUMMY_SP, FileMap}; use syntax_pos::hygiene::{Mark, SyntaxContext, ExpnInfo}; use ty; +use ty::maps::job::QueryResult; use ty::codec::{self as ty_codec, TyDecoder, TyEncoder}; use ty::context::TyCtxt; @@ -239,6 +240,10 @@ impl<'sess> OnDiskCache<'sess> { for (key, entry) in const_eval::get_cache_internal(tcx).map.iter() { use ty::maps::config::QueryDescription; if const_eval::cache_on_disk(key.clone()) { + let entry = match *entry { + QueryResult::Complete(ref v) => v, + _ => panic!("incomplete query"), + }; if let Ok(ref value) = entry.value { let dep_node = SerializedDepNodeIndex::new(entry.index.index()); @@ -1109,6 +1114,10 @@ fn encode_query_results<'enc, 'a, 'tcx, Q, E>(tcx: TyCtxt<'a, 'tcx, 'tcx>, { for (key, entry) in Q::get_cache_internal(tcx).map.iter() { if Q::cache_on_disk(key.clone()) { + let entry = match *entry { + QueryResult::Complete(ref v) => v, + _ => panic!("incomplete query"), + }; let dep_node = SerializedDepNodeIndex::new(entry.index.index()); // Record position of the cache entry diff --git a/src/librustc/ty/maps/plumbing.rs b/src/librustc/ty/maps/plumbing.rs index 46106d8ec0e91..895bf3d797336 100644 --- a/src/librustc/ty/maps/plumbing.rs +++ b/src/librustc/ty/maps/plumbing.rs @@ -15,19 +15,18 @@ use dep_graph::{DepNodeIndex, DepNode, DepKind, DepNodeColor}; use errors::DiagnosticBuilder; use ty::{TyCtxt}; -use ty::maps::Query; // NB: actually generated by the macros in this file use ty::maps::config::QueryDescription; +use ty::maps::job::{QueryResult, StackEntry}; use ty::item_path; use rustc_data_structures::fx::{FxHashMap}; -use std::cell::{Ref, RefMut}; +use rustc_data_structures::sync::LockGuard; use std::marker::PhantomData; -use std::mem; use syntax_pos::Span; pub(super) struct QueryMap<'tcx, D: QueryDescription<'tcx>> { phantom: PhantomData<(D, &'tcx ())>, - pub(super) map: FxHashMap>, + pub(super) map: FxHashMap>>, } pub(super) struct QueryValue { @@ -57,23 +56,19 @@ impl<'tcx, M: QueryDescription<'tcx>> QueryMap<'tcx, M> { pub(super) trait GetCacheInternal<'tcx>: QueryDescription<'tcx> + Sized { fn get_cache_internal<'a>(tcx: TyCtxt<'a, 'tcx, 'tcx>) - -> Ref<'a, QueryMap<'tcx, Self>>; + -> LockGuard<'a, QueryMap<'tcx, Self>>; } -pub(super) struct CycleError<'a, 'tcx: 'a> { - span: Span, - cycle: RefMut<'a, [(Span, Query<'tcx>)]>, +#[derive(Clone)] +pub(super) struct CycleError<'tcx> { + pub(super) span: Span, + pub(super) cycle: Vec>, } impl<'a, 'gcx, 'tcx> TyCtxt<'a, 'gcx, 'tcx> { - pub(super) fn report_cycle(self, CycleError { span, cycle }: CycleError) + pub(super) fn report_cycle(self, CycleError { span, cycle: stack }: CycleError) -> DiagnosticBuilder<'a> { - // Subtle: release the refcell lock before invoking `describe()` - // below by dropping `cycle`. - let stack = cycle.to_vec(); - mem::drop(cycle); - assert!(!stack.is_empty()); // Disable naming impls with types in this path, since that @@ -87,44 +82,21 @@ impl<'a, 'gcx, 'tcx> TyCtxt<'a, 'gcx, 'tcx> { "cyclic dependency detected"); err.span_label(span, "cyclic reference"); - err.span_note(self.sess.codemap().def_span(stack[0].0), - &format!("the cycle begins when {}...", stack[0].1.describe(self))); + err.span_note(self.sess.codemap().def_span(stack[0].span), + &format!("the cycle begins when {}...", stack[0].query.describe(self))); - for &(span, ref query) in &stack[1..] { + for &StackEntry { span, ref query, .. } in &stack[1..] { err.span_note(self.sess.codemap().def_span(span), &format!("...which then requires {}...", query.describe(self))); } err.note(&format!("...which then again requires {}, completing the cycle.", - stack[0].1.describe(self))); + stack[0].query.describe(self))); return err }) } - pub(super) fn cycle_check(self, span: Span, query: Query<'gcx>, compute: F) - -> Result> - where F: FnOnce() -> R - { - { - let mut stack = self.maps.query_stack.borrow_mut(); - if let Some((i, _)) = stack.iter().enumerate().rev() - .find(|&(_, &(_, ref q))| *q == query) { - return Err(CycleError { - span, - cycle: RefMut::map(stack, |stack| &mut stack[i..]) - }); - } - stack.push((span, query)); - } - - let result = compute(); - - self.maps.query_stack.borrow_mut().pop(); - - Ok(result) - } - /// Try to read a node index for the node dep_node. /// A node will have an index, when it's already been marked green, or when we can mark it /// green. This function will mark the current task as a reader of the specified node, when @@ -202,7 +174,11 @@ macro_rules! define_maps { [$($modifiers:tt)*] fn $name:ident: $node:ident($K:ty) -> $V:ty,)*) => { use dep_graph::DepNodeIndex; - use std::cell::RefCell; + use std::mem; + use errors::Diagnostic; + use errors::FatalError; + use rustc_data_structures::sync::{Lock, LockGuard}; + use rustc_data_structures::OnDrop; define_map_struct! { tcx: $tcx, @@ -214,8 +190,7 @@ macro_rules! define_maps { -> Self { Maps { providers, - query_stack: RefCell::new(vec![]), - $($name: RefCell::new(QueryMap::new())),* + $($name: Lock::new(QueryMap::new())),* } } } @@ -263,7 +238,7 @@ macro_rules! define_maps { impl<$tcx> GetCacheInternal<$tcx> for queries::$name<$tcx> { fn get_cache_internal<'a>(tcx: TyCtxt<'a, $tcx, $tcx>) - -> ::std::cell::Ref<'a, QueryMap<$tcx, Self>> { + -> LockGuard<'a, QueryMap<$tcx, Self>> { tcx.maps.$name.borrow() } } @@ -277,10 +252,47 @@ macro_rules! define_maps { DepNode::new(tcx, $node(*key)) } + fn try_get_lock(tcx: TyCtxt<'a, $tcx, 'lcx>, + mut span: Span, + key: &$K) + -> Result>, + Result<($V, DepNodeIndex), CycleError<$tcx>>> + { + loop { + let lock = tcx.maps.$name.borrow_mut(); + let job = if let Some(value) = lock.map.get(key) { + match *value { + QueryResult::Started(ref job) => Some(job.clone()), + QueryResult::Complete(ref value) => { + profq_msg!(tcx, ProfileQueriesMsg::CacheHit); + return Err(Ok(((&value.value).clone(), value.index))); + }, + QueryResult::Poisoned => FatalError.raise(), + } + } else { + None + }; + let job = if let Some(job) = job { + job + } else { + return Ok(lock); + }; + mem::drop(lock); + + if span == DUMMY_SP && stringify!($name) != "def_span" { + span = key.default_span(tcx); + } + + if let Err(cycle) = job.await(tcx, span) { + return Err(Err(cycle)); + } + } + } + fn try_get_with(tcx: TyCtxt<'a, $tcx, 'lcx>, mut span: Span, key: $K) - -> Result<$V, CycleError<'a, $tcx>> + -> Result<$V, CycleError<$tcx>> { debug!("ty::queries::{}::try_get_with(key={:?}, span={:?})", stringify!($name), @@ -294,24 +306,39 @@ macro_rules! define_maps { ) ); - if let Some(value) = tcx.maps.$name.borrow().map.get(&key) { - profq_msg!(tcx, ProfileQueriesMsg::CacheHit); - tcx.dep_graph.read_index(value.index); - return Ok((&value.value).clone()); + macro_rules! get_lock { + () => {{ + match Self::try_get_lock(tcx, span, &key) { + Ok(lock) => lock, + Err(result) => { + return result.map(|(v, index)| { + tcx.dep_graph.read_index(index); + v + }); + }, + } + }} } + let mut lock = get_lock!(); + // FIXME(eddyb) Get more valid Span's on queries. // def_span guard is necessary to prevent a recursive loop, // default_span calls def_span query internally. if span == DUMMY_SP && stringify!($name) != "def_span" { - span = key.default_span(tcx) + // This might deadlock if we hold the map lock since we might be + // waiting for the def_span query and switch to some other fiber + // So we drop the lock here and reacquire it + mem::drop(lock); + span = key.default_span(tcx); + lock = get_lock!(); } // Fast path for when incr. comp. is off. `to_dep_node` is // expensive for some DepKinds. if !tcx.dep_graph.is_fully_enabled() { let null_dep_node = DepNode::new_no_params(::dep_graph::DepKind::Null); - return Self::force(tcx, key, span, null_dep_node) + return Self::force_with_lock(tcx, key, span, lock, null_dep_node) .map(|(v, _)| v); } @@ -320,34 +347,36 @@ macro_rules! define_maps { if dep_node.kind.is_anon() { profq_msg!(tcx, ProfileQueriesMsg::ProviderBegin); - let res = tcx.cycle_check(span, Query::$name(key), || { - tcx.sess.diagnostic().track_diagnostics(|| { - tcx.dep_graph.with_anon_task(dep_node.kind, || { - Self::compute_result(tcx.global_tcx(), key) - }) + let res = Self::start_job(tcx, span, key, lock, |tcx| { + tcx.dep_graph.with_anon_task(dep_node.kind, || { + Self::compute_result(tcx.global_tcx(), key) }) })?; profq_msg!(tcx, ProfileQueriesMsg::ProviderEnd); - let ((result, dep_node_index), diagnostics) = res; + let (((result, dep_node_index), diagnostics), job) = res; tcx.dep_graph.read_index(dep_node_index); tcx.on_disk_query_result_cache .store_diagnostics_for_anon_node(dep_node_index, diagnostics); - let value = QueryValue::new(result, dep_node_index); + let value = QueryValue::new(Clone::clone(&result), dep_node_index); + + tcx.maps + .$name + .borrow_mut() + .map + .insert(key, QueryResult::Complete(value)); - return Ok((&tcx.maps - .$name - .borrow_mut() - .map - .entry(key) - .or_insert(value) - .value).clone()); + job.signal_complete(); + + return Ok(result); } if !dep_node.kind.is_input() { + // try_mark_green_and_read may force queries. So we must drop our lock here + mem::drop(lock); if let Some(dep_node_index) = tcx.try_mark_green_and_read(&dep_node) { profq_msg!(tcx, ProfileQueriesMsg::CacheHit); return Self::load_from_disk_and_cache_in_memory(tcx, @@ -356,9 +385,10 @@ macro_rules! define_maps { dep_node_index, &dep_node) } + lock = get_lock!(); } - match Self::force(tcx, key, span, dep_node) { + match Self::force_with_lock(tcx, key, span, lock, dep_node) { Ok((result, dep_node_index)) => { tcx.dep_graph.read_index(dep_node_index); Ok(result) @@ -391,6 +421,63 @@ macro_rules! define_maps { } } + fn start_job(tcx: TyCtxt<'_, $tcx, 'lcx>, + span: Span, + key: $K, + mut map: LockGuard<'_, QueryMap<$tcx, Self>>, + compute: F) + -> Result<((R, Vec), Lrc>), CycleError<$tcx>> + where F: for<'b> FnOnce(TyCtxt<'b, $tcx, 'lcx>) -> R + { + let query = Query::$name(Clone::clone(&key)); + + let entry = StackEntry { + span, + query, + }; + + let (r, job) = ty::tls::with_related_context(tcx, move |icx| { + let job = Lrc::new(QueryJob::new(entry, true, icx.query.clone())); + + map.map.entry(key).or_insert(QueryResult::Started(job.clone())); + + mem::drop(map); + + let r = { + let on_drop = OnDrop(|| { + // Poison the query so jobs waiting on it panics + tcx.maps + .$name + .borrow_mut() + .map + .insert(key, QueryResult::Poisoned); + // Also signal the completion of the job, so waiters + // will continue execution + job.signal_complete(); + }); + + let icx = ty::tls::ImplicitCtxt { + tcx, + query: Some(job.clone()), + }; + + let r = ty::tls::enter_context(&icx, |icx| { + compute(icx.tcx) + }); + + mem::forget(on_drop); + + r + }; + + (r, job) + }); + + let diagnostics: Vec<_> = mem::replace(&mut *job.diagnostics.lock(), Vec::new()); + + Ok(((r, diagnostics), job)) + } + fn compute_result(tcx: TyCtxt<'a, $tcx, 'lcx>, key: $K) -> $V { let provider = tcx.maps.providers[key.map_crate()].$name; provider(tcx.global_tcx(), key) @@ -401,8 +488,11 @@ macro_rules! define_maps { span: Span, dep_node_index: DepNodeIndex, dep_node: &DepNode) - -> Result<$V, CycleError<'a, $tcx>> + -> Result<$V, CycleError<$tcx>> { + // Note this function can be called concurrently from the same query + // We must ensure that this is handled correctly + debug_assert!(tcx.dep_graph.is_green(dep_node)); // First we try to load the result from the on-disk cache @@ -425,24 +515,27 @@ macro_rules! define_maps { None }; - let result = if let Some(result) = result { - result + let (result, job) = if let Some(result) = result { + (result, None) } else { // We could not load a result from the on-disk cache, so // recompute. - let (result, _ ) = tcx.cycle_check(span, Query::$name(key), || { - // The diagnostics for this query have already been - // promoted to the current session during - // try_mark_green(), so we can ignore them here. - tcx.sess.diagnostic().track_diagnostics(|| { - // The dep-graph for this computation is already in - // place - tcx.dep_graph.with_ignore(|| { - Self::compute_result(tcx, key) - }) + + // The diagnostics for this query have already been + // promoted to the current session during + // try_mark_green(), so we can ignore them here. + let ((result, _), job) = Self::start_job(tcx, + span, + key, + tcx.maps.$name.borrow_mut(), + |tcx| { + // The dep-graph for this computation is already in + // place + tcx.dep_graph.with_ignore(|| { + Self::compute_result(tcx, key) }) })?; - result + (result, Some(job)) }; // If -Zincremental-verify-ich is specified, re-hash results from @@ -475,43 +568,67 @@ macro_rules! define_maps { tcx.dep_graph.mark_loaded_from_cache(dep_node_index, true); } - let value = QueryValue::new(result, dep_node_index); + let value = QueryValue::new(Clone::clone(&result), dep_node_index); - Ok((&tcx.maps - .$name - .borrow_mut() - .map - .entry(key) - .or_insert(value) - .value).clone()) + tcx.maps + .$name + .borrow_mut() + .map + .insert(key, QueryResult::Complete(value)); + + job.map(|j| j.signal_complete()); + + Ok(result) } + #[allow(dead_code)] fn force(tcx: TyCtxt<'a, $tcx, 'lcx>, key: $K, span: Span, dep_node: DepNode) - -> Result<($V, DepNodeIndex), CycleError<'a, $tcx>> { + -> Result<($V, DepNodeIndex), CycleError<$tcx>> { + // We may be concurrently trying both execute and force a query + // Ensure that only one of them runs the query + let lock = match Self::try_get_lock(tcx, span, &key) { + Ok(lock) => lock, + Err(result) => return result, + }; + Self::force_with_lock(tcx, + key, + span, + lock, + dep_node) + } + + fn force_with_lock(tcx: TyCtxt<'a, $tcx, 'lcx>, + key: $K, + span: Span, + map: LockGuard<'_, QueryMap<$tcx, Self>>, + dep_node: DepNode) + -> Result<($V, DepNodeIndex), CycleError<$tcx>> { debug_assert!(!tcx.dep_graph.dep_node_exists(&dep_node)); profq_msg!(tcx, ProfileQueriesMsg::ProviderBegin); - let res = tcx.cycle_check(span, Query::$name(key), || { - tcx.sess.diagnostic().track_diagnostics(|| { - if dep_node.kind.is_eval_always() { - tcx.dep_graph.with_eval_always_task(dep_node, - tcx, - key, - Self::compute_result) - } else { - tcx.dep_graph.with_task(dep_node, - tcx, - key, - Self::compute_result) - } - }) + let res = Self::start_job(tcx, + span, + key, + map, + |tcx| { + if dep_node.kind.is_eval_always() { + tcx.dep_graph.with_eval_always_task(dep_node, + tcx, + key, + Self::compute_result) + } else { + tcx.dep_graph.with_task(dep_node, + tcx, + key, + Self::compute_result) + } })?; profq_msg!(tcx, ProfileQueriesMsg::ProviderEnd); - let ((result, dep_node_index), diagnostics) = res; + let (((result, dep_node_index), diagnostics), job) = res; if tcx.sess.opts.debugging_opts.query_dep_graph { tcx.dep_graph.mark_loaded_from_cache(dep_node_index, false); @@ -522,16 +639,19 @@ macro_rules! define_maps { .store_diagnostics(dep_node_index, diagnostics); } - let value = QueryValue::new(result, dep_node_index); + let value = QueryValue::new(Clone::clone(&result), dep_node_index); + + tcx.maps + .$name + .borrow_mut() + .map + .insert(key, QueryResult::Complete(value)); + + let job: Lrc = job; + + job.signal_complete(); - Ok(((&tcx.maps - .$name - .borrow_mut() - .map - .entry(key) - .or_insert(value) - .value).clone(), - dep_node_index)) + Ok((result, dep_node_index)) } pub fn try_get(tcx: TyCtxt<'a, $tcx, 'lcx>, span: Span, key: $K) @@ -599,8 +719,7 @@ macro_rules! define_map_struct { input: ($(([$($modifiers:tt)*] [$($attr:tt)*] [$name:ident]))*)) => { pub struct Maps<$tcx> { providers: IndexVec>, - query_stack: RefCell)>>, - $($(#[$attr])* $name: RefCell>>,)* + $($(#[$attr])* $name: Lock>>,)* } }; } diff --git a/src/librustc_errors/lib.rs b/src/librustc_errors/lib.rs index a25c3668bb13b..23a688636e997 100644 --- a/src/librustc_errors/lib.rs +++ b/src/librustc_errors/lib.rs @@ -42,7 +42,6 @@ use rustc_data_structures::stable_hasher::StableHasher; use std::borrow::Cow; use std::cell::{RefCell, Cell}; -use std::mem; use std::{error, fmt}; use std::sync::atomic::AtomicUsize; use std::sync::atomic::Ordering::SeqCst; @@ -269,7 +268,6 @@ pub struct Handler { emitter: RefCell>, continue_after_error: Cell, delayed_span_bug: RefCell>, - tracked_diagnostics: RefCell>>, // This set contains the `DiagnosticId` of all emitted diagnostics to avoid // emitting the same diagnostic with extended help (`--teach`) twice, which @@ -282,6 +280,11 @@ pub struct Handler { emitted_diagnostics: RefCell>, } +fn default_track_diagnostic(_: &Diagnostic) {} + +thread_local!(pub static TRACK_DIAGNOSTICS: Cell = + Cell::new(default_track_diagnostic)); + #[derive(Default)] pub struct HandlerFlags { pub can_emit_warnings: bool, @@ -333,7 +336,6 @@ impl Handler { emitter: RefCell::new(e), continue_after_error: Cell::new(true), delayed_span_bug: RefCell::new(None), - tracked_diagnostics: RefCell::new(None), tracked_diagnostic_codes: RefCell::new(FxHashSet()), emitted_diagnostics: RefCell::new(FxHashSet()), } @@ -629,17 +631,6 @@ impl Handler { } } - pub fn track_diagnostics(&self, f: F) -> (R, Vec) - where F: FnOnce() -> R - { - let prev = mem::replace(&mut *self.tracked_diagnostics.borrow_mut(), - Some(Vec::new())); - let ret = f(); - let diagnostics = mem::replace(&mut *self.tracked_diagnostics.borrow_mut(), prev) - .unwrap(); - (ret, diagnostics) - } - /// `true` if a diagnostic with this code has already been emitted in this handler. /// /// Used to suppress emitting the same error multiple times with extended explanation when @@ -651,9 +642,9 @@ impl Handler { fn emit_db(&self, db: &DiagnosticBuilder) { let diagnostic = &**db; - if let Some(ref mut list) = *self.tracked_diagnostics.borrow_mut() { - list.push(diagnostic.clone()); - } + TRACK_DIAGNOSTICS.with(|track_diagnostics| { + track_diagnostics.get()(diagnostic); + }); if let Some(ref code) = diagnostic.code { self.tracked_diagnostic_codes.borrow_mut().insert(code.clone()); From 4f7d0fde1c5f577c1f956d5d4edfbb202a5bc3cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?John=20K=C3=A5re=20Alsaker?= Date: Sat, 24 Mar 2018 06:19:20 +0100 Subject: [PATCH 02/12] Some cleanups and added comments --- src/librustc/ty/context.rs | 92 +++++++++++++++++++++++--------- src/librustc/ty/maps/job.rs | 69 ++++++++++++------------ src/librustc/ty/maps/mod.rs | 2 +- src/librustc/ty/maps/plumbing.rs | 80 ++++++++++++++++++--------- 4 files changed, 157 insertions(+), 86 deletions(-) diff --git a/src/librustc/ty/context.rs b/src/librustc/ty/context.rs index adb0ecd39846a..44b9d61cf0202 100644 --- a/src/librustc/ty/context.rs +++ b/src/librustc/ty/context.rs @@ -1489,10 +1489,13 @@ impl<'a, 'tcx> TyCtxt<'a, 'tcx, 'tcx> { impl<'gcx: 'tcx, 'tcx> GlobalCtxt<'gcx> { /// Call the closure with a local `TyCtxt` using the given arena. - pub fn enter_local(&self, - arena: &'tcx DroplessArena, - f: F) -> R - where F: for<'a> FnOnce(TyCtxt<'a, 'gcx, 'tcx>) -> R + pub fn enter_local( + &self, + arena: &'tcx DroplessArena, + f: F + ) -> R + where + F: for<'a> FnOnce(TyCtxt<'a, 'gcx, 'tcx>) -> R { let interners = CtxtInterners::new(arena); let tcx = TyCtxt { @@ -1665,12 +1668,23 @@ pub mod tls { use rustc_data_structures::OnDrop; use rustc_data_structures::sync::Lrc; + /// This is the implicit state of rustc. It contains the current + /// TyCtxt and query. It is updated when creating a local interner or + /// executing a new query. Whenever there's a TyCtxt value available + /// you should also have access to an ImplicitCtxt through the functions + /// in this module. #[derive(Clone)] pub struct ImplicitCtxt<'a, 'gcx: 'a+'tcx, 'tcx: 'a> { + /// The current TyCtxt. Initially created by `enter_global` and updated + /// by `enter_local` with a new local interner pub tcx: TyCtxt<'a, 'gcx, 'tcx>, + + /// The current query job, if any. This is updated by start_job in + /// ty::maps::plumbing when executing a query pub query: Option>>, } + // A thread local value which stores a pointer to the current ImplicitCtxt thread_local!(static TLV: Cell = Cell::new(0)); fn set_tlv R, R>(value: usize, f: F) -> R { @@ -1684,12 +1698,17 @@ pub mod tls { TLV.with(|tlv| tlv.get()) } + /// This is a callback from libsyntax as it cannot access the implicit state + /// in librustc otherwise fn span_debug(span: syntax_pos::Span, f: &mut fmt::Formatter) -> fmt::Result { with(|tcx| { write!(f, "{}", tcx.sess.codemap().span_to_string(span)) }) } + /// This is a callback from libsyntax as it cannot access the implicit state + /// in librustc otherwise. It is used to when diagnostic messages are + /// emitted and stores them in the current query, if there is one. fn track_diagnostic(diagnostic: &Diagnostic) { with_context(|context| { if let Some(ref query) = context.query { @@ -1698,6 +1717,7 @@ pub mod tls { }) } + /// Sets up the callbacks from libsyntax on the current thread pub fn with_thread_locals(f: F) -> R where F: FnOnce() -> R { @@ -1722,6 +1742,20 @@ pub mod tls { }) } + /// Sets `context` as the new current ImplicitCtxt for the duration of the function `f` + pub fn enter_context<'a, 'gcx: 'tcx, 'tcx, F, R>(context: &ImplicitCtxt<'a, 'gcx, 'tcx>, + f: F) -> R + where F: FnOnce(&ImplicitCtxt<'a, 'gcx, 'tcx>) -> R + { + set_tlv(context as *const _ as usize, || { + f(&context) + }) + } + + /// Enters GlobalCtxt by setting up libsyntax callbacks and + /// creating a initial TyCtxt and ImplicitCtxt. + /// This happens once per rustc session and TyCtxts only exists + /// inside the `f` function. pub fn enter_global<'gcx, F, R>(gcx: &GlobalCtxt<'gcx>, f: F) -> R where F: for<'a> FnOnce(TyCtxt<'a, 'gcx, 'gcx>) -> R { @@ -1740,15 +1774,7 @@ pub mod tls { }) } - pub fn enter_context<'a, 'gcx: 'tcx, 'tcx, F, R>(context: &ImplicitCtxt<'a, 'gcx, 'tcx>, - f: F) -> R - where F: FnOnce(&ImplicitCtxt<'a, 'gcx, 'tcx>) -> R - { - set_tlv(context as *const _ as usize, || { - f(&context) - }) - } - + /// Allows access to the current ImplicitCtxt in a closure if one is available pub fn with_context_opt(f: F) -> R where F: for<'a, 'gcx, 'tcx> FnOnce(Option<&ImplicitCtxt<'a, 'gcx, 'tcx>>) -> R { @@ -1760,46 +1786,62 @@ pub mod tls { } } - pub fn with_fully_related_context<'a, 'gcx, 'tcx, F, R>(tcx: TyCtxt<'a, 'gcx, 'tcx>, f: F) -> R - where F: for<'b> FnOnce(&ImplicitCtxt<'b, 'gcx, 'tcx>) -> R + /// Allows access to the current ImplicitCtxt. + /// Panics if there is no ImplicitCtxt available + pub fn with_context(f: F) -> R + where F: for<'a, 'gcx, 'tcx> FnOnce(&ImplicitCtxt<'a, 'gcx, 'tcx>) -> R + { + with_context_opt(|opt_context| f(opt_context.expect("no ImplicitCtxt stored in tls"))) + } + + /// Allows access to the current ImplicitCtxt whose tcx field has the same global + /// interner as the tcx argument passed in. This means the closure is given an ImplicitCtxt + /// with the same 'gcx lifetime as the TyCtxt passed in. + /// This will panic if you pass it a TyCtxt which has a different global interner from + /// the current ImplicitCtxt's tcx field. + pub fn with_related_context<'a, 'gcx, 'tcx1, F, R>(tcx: TyCtxt<'a, 'gcx, 'tcx1>, f: F) -> R + where F: for<'b, 'tcx2> FnOnce(&ImplicitCtxt<'b, 'gcx, 'tcx2>) -> R { with_context(|context| { unsafe { let gcx = tcx.gcx as *const _ as usize; - let interners = tcx.interners as *const _ as usize; assert!(context.tcx.gcx as *const _ as usize == gcx); - assert!(context.tcx.interners as *const _ as usize == interners); let context: &ImplicitCtxt = mem::transmute(context); f(context) } }) } - pub fn with_related_context<'a, 'gcx, 'tcx1, F, R>(tcx: TyCtxt<'a, 'gcx, 'tcx1>, f: F) -> R - where F: for<'b, 'tcx2> FnOnce(&ImplicitCtxt<'b, 'gcx, 'tcx2>) -> R + /// Allows access to the current ImplicitCtxt whose tcx field has the same global + /// interner and local interner as the tcx argument passed in. This means the closure + /// is given an ImplicitCtxt with the same 'tcx and 'gcx lifetimes as the TyCtxt passed in. + /// This will panic if you pass it a TyCtxt which has a different global interner or + /// a different local interner from the current ImplicitCtxt's tcx field. + pub fn with_fully_related_context<'a, 'gcx, 'tcx, F, R>(tcx: TyCtxt<'a, 'gcx, 'tcx>, f: F) -> R + where F: for<'b> FnOnce(&ImplicitCtxt<'b, 'gcx, 'tcx>) -> R { with_context(|context| { unsafe { let gcx = tcx.gcx as *const _ as usize; + let interners = tcx.interners as *const _ as usize; assert!(context.tcx.gcx as *const _ as usize == gcx); + assert!(context.tcx.interners as *const _ as usize == interners); let context: &ImplicitCtxt = mem::transmute(context); f(context) } }) } - pub fn with_context(f: F) -> R - where F: for<'a, 'gcx, 'tcx> FnOnce(&ImplicitCtxt<'a, 'gcx, 'tcx>) -> R - { - with_context_opt(|opt_context| f(opt_context.expect("no ImplicitCtxt stored in tls"))) - } - + /// Allows access to the TyCtxt in the current ImplicitCtxt. + /// Panics if there is no ImplicitCtxt available pub fn with(f: F) -> R where F: for<'a, 'gcx, 'tcx> FnOnce(TyCtxt<'a, 'gcx, 'tcx>) -> R { with_context(|context| f(context.tcx)) } + /// Allows access to the TyCtxt in the current ImplicitCtxt. + /// The closure is passed None if there is no ImplicitCtxt available pub fn with_opt(f: F) -> R where F: for<'a, 'gcx, 'tcx> FnOnce(Option>) -> R { diff --git a/src/librustc/ty/maps/job.rs b/src/librustc/ty/maps/job.rs index 2ed993bd975e5..7d756fb16a453 100644 --- a/src/librustc/ty/maps/job.rs +++ b/src/librustc/ty/maps/job.rs @@ -8,65 +8,69 @@ // option. This file may not be copied, modified, or distributed // except according to those terms. -#![allow(warnings)] - -use std::mem; -use std::sync::atomic::AtomicBool; -use std::sync::atomic::Ordering; -use rustc_data_structures::sync::{Lock, LockGuard, Lrc}; +use rustc_data_structures::sync::{Lock, Lrc}; use syntax_pos::Span; use ty::tls; use ty::maps::Query; use ty::maps::plumbing::CycleError; use ty::context::TyCtxt; use errors::Diagnostic; -use std::process; -use std::fmt; -use std::sync::{Arc, Mutex}; -use std::collections::HashSet; -pub struct PoisonedJob; +/// Indicates the state of a query for a given key in a query map +pub(super) enum QueryResult<'tcx, T> { + /// An already executing query. The query job can be used to await for its completion + Started(Lrc>), + + /// The query is complete and produced `T` + Complete(T), + /// The query panicked. Queries trying to wait on this will raise a fatal error / silently panic + Poisoned, +} + +/// A span and a query key #[derive(Clone, Debug)] -pub struct StackEntry<'tcx> { +pub struct QueryInfo<'tcx> { pub span: Span, pub query: Query<'tcx>, } +/// A object representing an active query job. pub struct QueryJob<'tcx> { - pub entry: StackEntry<'tcx>, + pub info: QueryInfo<'tcx>, + + /// The parent query job which created this job and is implicitly waiting on it. pub parent: Option>>, - pub track_diagnostics: bool, + + /// Diagnostic messages which are emitted while the query executes pub diagnostics: Lock>, } impl<'tcx> QueryJob<'tcx> { - pub fn new( - entry: StackEntry<'tcx>, - track_diagnostics: bool, - parent: Option>>, - ) -> Self { + /// Creates a new query job + pub fn new(info: QueryInfo<'tcx>, parent: Option>>) -> Self { QueryJob { - track_diagnostics, diagnostics: Lock::new(Vec::new()), - entry, + info, parent, } } + /// Awaits for the query job to complete. + /// + /// For single threaded rustc there's no concurrent jobs running, so if we are waiting for any + /// query that means that there is a query cycle, thus this always running a cycle error. pub(super) fn await<'lcx>( &self, tcx: TyCtxt<'_, 'tcx, 'lcx>, span: Span, ) -> Result<(), CycleError<'tcx>> { - // The query is already executing, so this must be a cycle for single threaded rustc, - // so we find the cycle and return it - + // Get the current executing query (waiter) and find the waitee amongst its parents let mut current_job = tls::with_related_context(tcx, |icx| icx.query.clone()); let mut cycle = Vec::new(); while let Some(job) = current_job { - cycle.insert(0, job.entry.clone()); + cycle.insert(0, job.info.clone()); if &*job as *const _ == self as *const _ { break; @@ -78,14 +82,9 @@ impl<'tcx> QueryJob<'tcx> { Err(CycleError { span, cycle }) } - pub fn signal_complete(&self) { - // Signals to waiters that the query is complete. - // This is a no-op for single threaded rustc - } -} - -pub(super) enum QueryResult<'tcx, T> { - Started(Lrc>), - Complete(T), - Poisoned, + /// Signals to waiters that the query is complete. + /// + /// This does nothing for single threaded rustc, + /// as there are no concurrent jobs which could be waiting on us + pub fn signal_complete(&self) {} } diff --git a/src/librustc/ty/maps/mod.rs b/src/librustc/ty/maps/mod.rs index 6dd31baa5931b..f4977be78776b 100644 --- a/src/librustc/ty/maps/mod.rs +++ b/src/librustc/ty/maps/mod.rs @@ -67,7 +67,7 @@ use self::plumbing::*; pub use self::plumbing::force_from_dep_node; mod job; -pub use self::job::{QueryJob, StackEntry, PoisonedJob}; +pub use self::job::{QueryJob, QueryInfo}; use self::job::QueryResult; mod keys; diff --git a/src/librustc/ty/maps/plumbing.rs b/src/librustc/ty/maps/plumbing.rs index 895bf3d797336..c21b53cd427c0 100644 --- a/src/librustc/ty/maps/plumbing.rs +++ b/src/librustc/ty/maps/plumbing.rs @@ -16,7 +16,7 @@ use dep_graph::{DepNodeIndex, DepNode, DepKind, DepNodeColor}; use errors::DiagnosticBuilder; use ty::{TyCtxt}; use ty::maps::config::QueryDescription; -use ty::maps::job::{QueryResult, StackEntry}; +use ty::maps::job::{QueryResult, QueryInfo}; use ty::item_path; use rustc_data_structures::fx::{FxHashMap}; @@ -62,7 +62,18 @@ pub(super) trait GetCacheInternal<'tcx>: QueryDescription<'tcx> + Sized { #[derive(Clone)] pub(super) struct CycleError<'tcx> { pub(super) span: Span, - pub(super) cycle: Vec>, + pub(super) cycle: Vec>, +} + +/// The result of `try_get_lock` +pub(super) enum TryGetLock<'a, 'tcx: 'a, T, D: QueryDescription<'tcx> + 'a> { + /// The query is not yet started. Contains a guard to the map eventually used to start it. + NotYetStarted(LockGuard<'a, QueryMap<'tcx, D>>), + + /// The query was already completed. + /// Returns the result of the query and its dep node index + /// if it succeeded or a cycle error if it failed + JobCompleted(Result<(T, DepNodeIndex), CycleError<'tcx>>), } impl<'a, 'gcx, 'tcx> TyCtxt<'a, 'gcx, 'tcx> { @@ -85,7 +96,7 @@ impl<'a, 'gcx, 'tcx> TyCtxt<'a, 'gcx, 'tcx> { err.span_note(self.sess.codemap().def_span(stack[0].span), &format!("the cycle begins when {}...", stack[0].query.describe(self))); - for &StackEntry { span, ref query, .. } in &stack[1..] { + for &QueryInfo { span, ref query, .. } in &stack[1..] { err.span_note(self.sess.codemap().def_span(span), &format!("...which then requires {}...", query.describe(self))); } @@ -252,11 +263,14 @@ macro_rules! define_maps { DepNode::new(tcx, $node(*key)) } - fn try_get_lock(tcx: TyCtxt<'a, $tcx, 'lcx>, - mut span: Span, - key: &$K) - -> Result>, - Result<($V, DepNodeIndex), CycleError<$tcx>>> + /// Either get the lock of the query map, allowing us to + /// start executing the query, or it returns with the result of the query. + /// If the query already executed and panicked, this will fatal error / silently panic + fn try_get_lock( + tcx: TyCtxt<'a, $tcx, 'lcx>, + mut span: Span, + key: &$K + ) -> TryGetLock<'a, $tcx, $V, Self> { loop { let lock = tcx.maps.$name.borrow_mut(); @@ -265,7 +279,8 @@ macro_rules! define_maps { QueryResult::Started(ref job) => Some(job.clone()), QueryResult::Complete(ref value) => { profq_msg!(tcx, ProfileQueriesMsg::CacheHit); - return Err(Ok(((&value.value).clone(), value.index))); + let result = Ok(((&value.value).clone(), value.index)); + return TryGetLock::JobCompleted(result); }, QueryResult::Poisoned => FatalError.raise(), } @@ -275,16 +290,19 @@ macro_rules! define_maps { let job = if let Some(job) = job { job } else { - return Ok(lock); + return TryGetLock::NotYetStarted(lock); }; mem::drop(lock); + // This just matches the behavior of `try_get_with` so the span when + // we await matches the span we would use when executing. + // See the FIXME there. if span == DUMMY_SP && stringify!($name) != "def_span" { span = key.default_span(tcx); } if let Err(cycle) = job.await(tcx, span) { - return Err(Err(cycle)); + return TryGetLock::JobCompleted(Err(cycle)); } } } @@ -306,21 +324,23 @@ macro_rules! define_maps { ) ); - macro_rules! get_lock { + /// Get the lock used to start the query or + /// return the result of the completed query + macro_rules! get_lock_or_return { () => {{ match Self::try_get_lock(tcx, span, &key) { - Ok(lock) => lock, - Err(result) => { + TryGetLock::NotYetStarted(lock) => lock, + TryGetLock::JobCompleted(result) => { return result.map(|(v, index)| { tcx.dep_graph.read_index(index); v - }); - }, + }) + } } }} } - let mut lock = get_lock!(); + let mut lock = get_lock_or_return!(); // FIXME(eddyb) Get more valid Span's on queries. // def_span guard is necessary to prevent a recursive loop, @@ -331,7 +351,7 @@ macro_rules! define_maps { // So we drop the lock here and reacquire it mem::drop(lock); span = key.default_span(tcx); - lock = get_lock!(); + lock = get_lock_or_return!(); } // Fast path for when incr. comp. is off. `to_dep_node` is @@ -385,7 +405,7 @@ macro_rules! define_maps { dep_node_index, &dep_node) } - lock = get_lock!(); + lock = get_lock_or_return!(); } match Self::force_with_lock(tcx, key, span, lock, dep_node) { @@ -421,6 +441,9 @@ macro_rules! define_maps { } } + /// Creates a job for the query and updates the query map indicating that it started. + /// Then it changes ImplicitCtxt to point to the new query job while it executes. + /// If the query panics, this updates the query map to indicate so. fn start_job(tcx: TyCtxt<'_, $tcx, 'lcx>, span: Span, key: $K, @@ -431,21 +454,25 @@ macro_rules! define_maps { { let query = Query::$name(Clone::clone(&key)); - let entry = StackEntry { + let entry = QueryInfo { span, query, }; + // The TyCtxt stored in TLS has the same global interner lifetime + // as `tcx`, so we use `with_related_context` to relate the 'gcx lifetimes + // when accessing the ImplicitCtxt let (r, job) = ty::tls::with_related_context(tcx, move |icx| { - let job = Lrc::new(QueryJob::new(entry, true, icx.query.clone())); + let job = Lrc::new(QueryJob::new(entry, icx.query.clone())); + // Store the job in the query map and drop the lock to allow + // others to wait it map.map.entry(key).or_insert(QueryResult::Started(job.clone())); - mem::drop(map); let r = { let on_drop = OnDrop(|| { - // Poison the query so jobs waiting on it panics + // Poison the query so jobs waiting on it panic tcx.maps .$name .borrow_mut() @@ -456,11 +483,13 @@ macro_rules! define_maps { job.signal_complete(); }); + // Update the ImplicitCtxt to point to our new query job let icx = ty::tls::ImplicitCtxt { tcx, query: Some(job.clone()), }; + // Use the ImplicitCtxt while we execute the query let r = ty::tls::enter_context(&icx, |icx| { compute(icx.tcx) }); @@ -473,6 +502,7 @@ macro_rules! define_maps { (r, job) }); + // Extract the diagnostic from the job let diagnostics: Vec<_> = mem::replace(&mut *job.diagnostics.lock(), Vec::new()); Ok(((r, diagnostics), job)) @@ -590,8 +620,8 @@ macro_rules! define_maps { // We may be concurrently trying both execute and force a query // Ensure that only one of them runs the query let lock = match Self::try_get_lock(tcx, span, &key) { - Ok(lock) => lock, - Err(result) => return result, + TryGetLock::NotYetStarted(lock) => lock, + TryGetLock::JobCompleted(result) => return result, }; Self::force_with_lock(tcx, key, From 71dc1626bddc9c966156cd8894d91845bc9aab9f Mon Sep 17 00:00:00 2001 From: scalexm Date: Wed, 28 Mar 2018 14:13:08 +0200 Subject: [PATCH 03/12] Tweak `Clause` definition and HRTBs --- src/librustc/ich/impls_ty.rs | 12 +++--- src/librustc/traits/mod.rs | 35 +++++++++++++---- src/librustc/traits/structural_impls.rs | 42 ++++++++++++-------- src/librustc_traits/lowering.rs | 52 +++++++++++++------------ 4 files changed, 88 insertions(+), 53 deletions(-) diff --git a/src/librustc/ich/impls_ty.rs b/src/librustc/ich/impls_ty.rs index 9a442e0529938..dfc64dfdb27e9 100644 --- a/src/librustc/ich/impls_ty.rs +++ b/src/librustc/ich/impls_ty.rs @@ -1392,6 +1392,12 @@ impl<'a, 'tcx> HashStable> for traits::Goal<'tcx> { } } +impl_stable_hash_for!( + impl<'tcx> for struct traits::ProgramClause<'tcx> { + goal, hypotheses + } +); + impl<'a, 'tcx> HashStable> for traits::Clause<'tcx> { fn hash_stable(&self, hcx: &mut StableHashingContext<'a>, @@ -1400,11 +1406,7 @@ impl<'a, 'tcx> HashStable> for traits::Clause<'tcx> { mem::discriminant(self).hash_stable(hcx, hasher); match self { - Implies(hypotheses, goal) => { - hypotheses.hash_stable(hcx, hasher); - goal.hash_stable(hcx, hasher); - } - DomainGoal(domain_goal) => domain_goal.hash_stable(hcx, hasher), + Implies(clause) => clause.hash_stable(hcx, hasher), ForAll(clause) => clause.hash_stable(hcx, hasher), } } diff --git a/src/librustc/traits/mod.rs b/src/librustc/traits/mod.rs index 1d5d3e41c9c9a..33930f4a59992 100644 --- a/src/librustc/traits/mod.rs +++ b/src/librustc/traits/mod.rs @@ -272,6 +272,8 @@ pub enum DomainGoal<'tcx> { TypeOutlives(ty::TypeOutlivesPredicate<'tcx>), } +pub type PolyDomainGoal<'tcx> = ty::Binder>; + #[derive(Copy, Clone, PartialEq, Eq, Hash, Debug)] pub enum QuantifierKind { Universal, @@ -294,9 +296,15 @@ impl<'tcx> From> for Goal<'tcx> { } } -impl<'tcx> From> for Clause<'tcx> { - fn from(domain_goal: DomainGoal<'tcx>) -> Self { - Clause::DomainGoal(domain_goal) +impl<'tcx> From> for Goal<'tcx> { + fn from(domain_goal: PolyDomainGoal<'tcx>) -> Self { + match domain_goal.no_late_bound_regions() { + Some(p) => p.into(), + None => Goal::Quantified( + QuantifierKind::Universal, + Box::new(domain_goal.map_bound(|p| p.into())) + ), + } } } @@ -304,10 +312,23 @@ impl<'tcx> From> for Clause<'tcx> { /// Harrop Formulas". #[derive(Clone, PartialEq, Eq, Hash, Debug)] pub enum Clause<'tcx> { - // FIXME: again, use interned refs instead of `Box` - Implies(Vec>, DomainGoal<'tcx>), - DomainGoal(DomainGoal<'tcx>), - ForAll(Box>>), + Implies(ProgramClause<'tcx>), + ForAll(ty::Binder>), +} + +/// A "program clause" has the form `D :- G1, ..., Gn`. It is saying +/// that the domain goal `D` is true if `G1...Gn` are provable. This +/// is equivalent to the implication `G1..Gn => D`; we usually write +/// it with the reverse implication operator `:-` to emphasize the way +/// that programs are actually solved (via backchaining, which starts +/// with the goal to solve and proceeds from there). +#[derive(Clone, PartialEq, Eq, Hash, Debug)] +pub struct ProgramClause<'tcx> { + /// This goal will be considered true... + pub goal: DomainGoal<'tcx>, + + /// ...if we can prove these hypotheses (there may be no hypotheses at all): + pub hypotheses: Vec>, } pub type Selection<'tcx> = Vtable<'tcx, PredicateObligation<'tcx>>; diff --git a/src/librustc/traits/structural_impls.rs b/src/librustc/traits/structural_impls.rs index d6e6f0e98adc4..865a9a34aaa25 100644 --- a/src/librustc/traits/structural_impls.rs +++ b/src/librustc/traits/structural_impls.rs @@ -493,25 +493,29 @@ impl<'tcx> fmt::Display for traits::Goal<'tcx> { } } +impl<'tcx> fmt::Display for traits::ProgramClause<'tcx> { + fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { + let traits::ProgramClause { goal, hypotheses } = self; + write!(fmt, "{}", goal)?; + if !hypotheses.is_empty() { + write!(fmt, " :- ")?; + for (index, condition) in hypotheses.iter().enumerate() { + if index > 0 { + write!(fmt, ", ")?; + } + write!(fmt, "{}", condition)?; + } + } + write!(fmt, ".") + } +} + impl<'tcx> fmt::Display for traits::Clause<'tcx> { fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { use traits::Clause::*; match self { - Implies(hypotheses, goal) => { - write!(fmt, "{}", goal)?; - if !hypotheses.is_empty() { - write!(fmt, " :- ")?; - for (index, condition) in hypotheses.iter().enumerate() { - if index > 0 { - write!(fmt, ", ")?; - } - write!(fmt, "{}", condition)?; - } - } - write!(fmt, ".") - } - DomainGoal(domain_goal) => write!(fmt, "{}.", domain_goal), + Implies(clause) => write!(fmt, "{}", clause), ForAll(clause) => { // FIXME: appropriate binder names write!(fmt, "forall<> {{ {} }}", clause.skip_binder()) @@ -553,10 +557,16 @@ EnumTypeFoldableImpl! { } } +BraceStructTypeFoldableImpl! { + impl<'tcx> TypeFoldable<'tcx> for traits::ProgramClause<'tcx> { + goal, + hypotheses + } +} + EnumTypeFoldableImpl! { impl<'tcx> TypeFoldable<'tcx> for traits::Clause<'tcx> { - (traits::Clause::Implies)(hypotheses, goal), - (traits::Clause::DomainGoal)(domain_goal), + (traits::Clause::Implies)(clause), (traits::Clause::ForAll)(clause), } } diff --git a/src/librustc_traits/lowering.rs b/src/librustc_traits/lowering.rs index 1092e826a35f9..4b1983d18d6a5 100644 --- a/src/librustc_traits/lowering.rs +++ b/src/librustc_traits/lowering.rs @@ -13,7 +13,7 @@ use rustc::hir::def_id::DefId; use rustc::hir::intravisit::{self, NestedVisitorMap, Visitor}; use rustc::ty::{self, TyCtxt}; use rustc::ty::subst::Substs; -use rustc::traits::{QuantifierKind, Goal, DomainGoal, Clause, WhereClauseAtom}; +use rustc::traits::{WhereClauseAtom, PolyDomainGoal, DomainGoal, ProgramClause, Clause}; use syntax::ast; use rustc_data_structures::sync::Lrc; @@ -61,28 +61,19 @@ impl<'tcx> Lower> for ty::TypeOutlivesPredicate<'tcx> { /// `ty::Binder` is used for wrapping a rustc construction possibly containing generic /// lifetimes, e.g. `for<'a> T: Fn(&'a i32)`. Instead of representing higher-ranked things /// in that leaf-form (i.e. `Holds(Implemented(Binder))` in the previous -/// example), we model them with quantified goals, e.g. as for the previous example: +/// example), we model them with quantified domain goals, e.g. as for the previous example: /// `forall<'a> { T: Fn(&'a i32) }` which corresponds to something like /// `Binder`. -/// -/// Also, if `self` does not contain generic lifetimes, we can safely drop the binder and we -/// can directly lower to a leaf goal instead of a quantified goal. -impl<'tcx, T> Lower> for ty::Binder - where T: Lower> + ty::fold::TypeFoldable<'tcx> + Copy +impl<'tcx, T> Lower> for ty::Binder + where T: Lower> + ty::fold::TypeFoldable<'tcx> { - fn lower(&self) -> Goal<'tcx> { - match self.no_late_bound_regions() { - Some(p) => p.lower().into(), - None => Goal::Quantified( - QuantifierKind::Universal, - Box::new(self.map_bound(|p| p.lower().into())) - ), - } + fn lower(&self) -> PolyDomainGoal<'tcx> { + self.map_bound_ref(|p| p.lower()) } } -impl<'tcx> Lower> for ty::Predicate<'tcx> { - fn lower(&self) -> Goal<'tcx> { +impl<'tcx> Lower> for ty::Predicate<'tcx> { + fn lower(&self) -> PolyDomainGoal<'tcx> { use rustc::ty::Predicate::*; match self { @@ -90,7 +81,7 @@ impl<'tcx> Lower> for ty::Predicate<'tcx> { RegionOutlives(predicate) => predicate.lower(), TypeOutlives(predicate) => predicate.lower(), Projection(predicate) => predicate.lower(), - WellFormed(ty) => DomainGoal::WellFormedTy(*ty).into(), + WellFormed(ty) => ty::Binder::dummy(DomainGoal::WellFormedTy(*ty)), ObjectSafe(..) | ClosureKind(..) | Subtype(..) | @@ -134,13 +125,16 @@ fn program_clauses_for_trait<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>, def_id: DefI } }; // `FromEnv(Self: Trait)` - let from_env = Goal::DomainGoal(DomainGoal::FromEnv(trait_pred.lower())); + let from_env = DomainGoal::FromEnv(trait_pred.lower()).into(); // `Implemented(Self: Trait)` let impl_trait = DomainGoal::Holds(WhereClauseAtom::Implemented(trait_pred)); // `Implemented(Self: Trait) :- FromEnv(Self: Trait)` - let clause = Clause::Implies(vec![from_env], impl_trait); - Lrc::new(vec![clause]) + let clause = ProgramClause { + goal: impl_trait, + hypotheses: vec![from_env], + }; + Lrc::new(vec![Clause::ForAll(ty::Binder::dummy(clause))]) } fn program_clauses_for_impl<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>, def_id: DefId) @@ -167,8 +161,11 @@ fn program_clauses_for_impl<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>, def_id: DefId let where_clauses = tcx.predicates_of(def_id).predicates.lower(); // `Implemented(A0: Trait) :- WC` - let clause = Clause::Implies(where_clauses, trait_pred); - Lrc::new(vec![clause]) + let clause = ProgramClause { + goal: trait_pred, + hypotheses: where_clauses.into_iter().map(|wc| wc.into()).collect() + }; + Lrc::new(vec![Clause::ForAll(ty::Binder::dummy(clause))]) } pub fn dump_program_clauses<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>) { @@ -184,14 +181,19 @@ struct ClauseDumper<'a, 'tcx: 'a> { tcx: TyCtxt<'a, 'tcx, 'tcx>, } -impl <'a, 'tcx> ClauseDumper<'a, 'tcx > { +impl<'a, 'tcx> ClauseDumper<'a, 'tcx > { fn process_attrs(&mut self, node_id: ast::NodeId, attrs: &[ast::Attribute]) { let def_id = self.tcx.hir.local_def_id(node_id); for attr in attrs { if attr.check_name("rustc_dump_program_clauses") { let clauses = self.tcx.program_clauses_for(def_id); for clause in &*clauses { - self.tcx.sess.struct_span_err(attr.span, &format!("{}", clause)).emit(); + let program_clause = match clause { + Clause::Implies(program_clause) => program_clause, + Clause::ForAll(program_clause) => program_clause.skip_binder(), + }; + // Skip the top-level binder for a less verbose output + self.tcx.sess.struct_span_err(attr.span, &format!("{}", program_clause)).emit(); } } } From 1074a22905c6c96f2b7013073510d5f57a555436 Mon Sep 17 00:00:00 2001 From: Alexandre Martin Date: Mon, 2 Apr 2018 22:25:22 +0200 Subject: [PATCH 04/12] Fix comment --- src/librustc_traits/lowering.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/librustc_traits/lowering.rs b/src/librustc_traits/lowering.rs index 4b1983d18d6a5..153b2e730337d 100644 --- a/src/librustc_traits/lowering.rs +++ b/src/librustc_traits/lowering.rs @@ -188,11 +188,11 @@ impl<'a, 'tcx> ClauseDumper<'a, 'tcx > { if attr.check_name("rustc_dump_program_clauses") { let clauses = self.tcx.program_clauses_for(def_id); for clause in &*clauses { + // Skip the top-level binder for a less verbose output let program_clause = match clause { Clause::Implies(program_clause) => program_clause, Clause::ForAll(program_clause) => program_clause.skip_binder(), }; - // Skip the top-level binder for a less verbose output self.tcx.sess.struct_span_err(attr.span, &format!("{}", program_clause)).emit(); } } From 553c04d9eb311189cbd01d1af7f6c2c26578342c Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Mon, 2 Apr 2018 08:19:32 -0700 Subject: [PATCH 05/12] proc_macro: Reorganize public API This commit is a reorganization of the `proc_macro` crate's public user-facing API. This is the result of a number of discussions at the recent Rust All-Hands where we're hoping to get the `proc_macro` crate into ship shape for stabilization of a subset of its functionality in the Rust 2018 release. The reorganization here is motivated by experiences from the `proc-macro2`, `quote`, and `syn` crates on crates.io (and other crates which depend on them). The main focus is future flexibility along with making a few more operations consistent and/or fixing bugs. A summary of the changes made from today's `proc_macro` API is: * The `TokenNode` enum has been removed and the public fields of `TokenTree` have also been removed. Instead the `TokenTree` type is now a public enum (what `TokenNode` was) and each variant is an opaque struct which internally contains `Span` information. This makes the various tokens a bit more consistent, require fewer wrappers, and otherwise provides good future-compatibility as opaque structs are easy to modify later on. * `Literal` integer constructors have been expanded to be unambiguous as to what they're doing and also allow for more future flexibility. Previously constructors like `Literal::float` and `Literal::integer` were used to create unsuffixed literals and the concrete methods like `Literal::i32` would create a suffixed token. This wasn't immediately clear to all users (the suffixed/unsuffixed aspect) and having *one* constructor for unsuffixed literals required us to pick a largest type which may not always be true. To fix these issues all constructors are now of the form `Literal::i32_unsuffixed` or `Literal::i32_suffixed` (for all integral types). This should allow future compatibility as well as being immediately clear what's suffixed and what isn't. * Each variant of `TokenTree` internally contains a `Span` which can also be configured via `set_span`. For example `Literal` and `Term` now both internally contain a `Span` rather than having it stored in an auxiliary location. * Constructors of all tokens are called `new` now (aka `Term::intern` is gone) and most do not take spans. Manufactured tokens typically don't have a fresh span to go with them and the span is purely used for error-reporting **except** the span for `Term`, which currently affects hygiene. The default spans for all these constructed tokens is `Span::call_site()` for now. The `Term` type's constructor explicitly requires passing in a `Span` to provide future-proofing against possible hygiene changes. It's intended that a first pass of stabilization will likely only stabilize `Span::call_site()` which is an explicit opt-in for "I would like no hygiene here please". The intention here is to make this explicit in procedural macros to be forwards-compatible with a hygiene-specifying solution. * Some of the conversions for `TokenStream` have been simplified a little. * The `TokenTreeIter` iterator was renamed to `token_stream::IntoIter`. Overall the hope is that this is the "final pass" at the API of `TokenStream` and most of `TokenTree` before stabilization. Explicitly left out here is any changes to `Span`'s API which will likely need to be re-evaluated before stabilization. All changes in this PR have already been reflected to the [`proc-macro2`], `quote`, and `syn` crates. New versions of all these crates have also been published to crates.io. Once this lands in nightly I plan on making an internals post again summarizing the changes made here and also calling on all macro authors to give the APIs a spin and see how they work. Hopefully pending no major issues we can then have an FCP to stabilize later this cycle! [`proc-macro2`]: https://docs.rs/proc-macro2/0.3.1/proc_macro2/ --- src/libproc_macro/lib.rs | 747 +++++++++++++----- src/libproc_macro/quote.rs | 138 ++-- .../auxiliary/attributes-included.rs | 87 +- .../auxiliary/cond_plugin.rs | 12 +- .../auxiliary/proc_macro_def.rs | 2 +- .../auxiliary/count_compound_ops.rs | 15 +- .../proc-macro/auxiliary/negative-token.rs | 10 +- .../proc-macro/auxiliary/span-api-tests.rs | 4 +- .../auxiliary/parent-source-spans.rs | 8 +- .../proc-macro/auxiliary/three-equals.rs | 17 +- 10 files changed, 726 insertions(+), 314 deletions(-) diff --git a/src/libproc_macro/lib.rs b/src/libproc_macro/lib.rs index 007093981d3e1..837900f05d2e7 100644 --- a/src/libproc_macro/lib.rs +++ b/src/libproc_macro/lib.rs @@ -59,7 +59,6 @@ use syntax::errors::DiagnosticBuilder; use syntax::parse::{self, token}; use syntax::symbol::Symbol; use syntax::tokenstream; -use syntax_pos::DUMMY_SP; use syntax_pos::{FileMap, Pos, SyntaxContext, FileName}; use syntax_pos::hygiene::Mark; @@ -73,7 +72,7 @@ use syntax_pos::hygiene::Mark; /// The API of this type is intentionally bare-bones, but it'll be expanded over /// time! #[stable(feature = "proc_macro_lib", since = "1.15.0")] -#[derive(Clone, Debug)] +#[derive(Clone)] pub struct TokenStream(tokenstream::TokenStream); /// Error returned from `TokenStream::from_str`. @@ -83,6 +82,20 @@ pub struct LexError { _inner: (), } +impl TokenStream { + /// Returns an empty `TokenStream`. + #[unstable(feature = "proc_macro", issue = "38356")] + pub fn empty() -> TokenStream { + TokenStream(tokenstream::TokenStream::empty()) + } + + /// Checks if this `TokenStream` is empty. + #[unstable(feature = "proc_macro", issue = "38356")] + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } +} + #[stable(feature = "proc_macro_lib", since = "1.15.0")] impl FromStr for TokenStream { type Err = LexError; @@ -110,19 +123,12 @@ impl fmt::Display for TokenStream { } } -/// `quote!(..)` accepts arbitrary tokens and expands into a `TokenStream` describing the input. -/// For example, `quote!(a + b)` will produce a expression, that, when evaluated, constructs -/// the `TokenStream` `[Word("a"), Op('+', Alone), Word("b")]`. -/// -/// Unquoting is done with `$`, and works by taking the single next ident as the unquoted term. -/// To quote `$` itself, use `$$`. -#[unstable(feature = "proc_macro", issue = "38356")] -#[macro_export] -macro_rules! quote { () => {} } - -#[unstable(feature = "proc_macro_internals", issue = "27812")] -#[doc(hidden)] -mod quote; +#[stable(feature = "proc_macro_lib", since = "1.15.0")] +impl fmt::Debug for TokenStream { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + self.0.fmt(f) + } +} #[unstable(feature = "proc_macro", issue = "38356")] impl From for TokenStream { @@ -132,62 +138,79 @@ impl From for TokenStream { } #[unstable(feature = "proc_macro", issue = "38356")] -impl From for TokenStream { - fn from(kind: TokenNode) -> TokenStream { - TokenTree::from(kind).into() - } -} - -#[unstable(feature = "proc_macro", issue = "38356")] -impl> iter::FromIterator for TokenStream { - fn from_iter>(streams: I) -> Self { +impl iter::FromIterator for TokenStream { + fn from_iter>(trees: I) -> Self { let mut builder = tokenstream::TokenStreamBuilder::new(); - for stream in streams { - builder.push(stream.into().0); + for tree in trees { + builder.push(tree.to_internal()); } TokenStream(builder.build()) } } +/// Implementation details for the `TokenTree` type, such as iterators. #[unstable(feature = "proc_macro", issue = "38356")] -impl IntoIterator for TokenStream { - type Item = TokenTree; - type IntoIter = TokenTreeIter; +pub mod token_stream { + use syntax::tokenstream; + use syntax_pos::DUMMY_SP; + + use {TokenTree, TokenStream, Delimiter}; - fn into_iter(self) -> TokenTreeIter { - TokenTreeIter { cursor: self.0.trees(), stack: Vec::new() } + /// An iterator over `TokenTree`s. + #[derive(Clone)] + #[unstable(feature = "proc_macro", issue = "38356")] + pub struct IntoIter { + cursor: tokenstream::Cursor, + stack: Vec, } -} -impl TokenStream { - /// Returns an empty `TokenStream`. #[unstable(feature = "proc_macro", issue = "38356")] - pub fn empty() -> TokenStream { - TokenStream(tokenstream::TokenStream::empty()) + impl Iterator for IntoIter { + type Item = TokenTree; + + fn next(&mut self) -> Option { + loop { + let tree = self.stack.pop().or_else(|| { + let next = self.cursor.next_as_stream()?; + Some(TokenTree::from_internal(next, &mut self.stack)) + })?; + if tree.span().0 == DUMMY_SP { + if let TokenTree::Group(ref group) = tree { + if group.delimiter() == Delimiter::None { + self.cursor.insert(group.stream.clone().0); + continue + } + } + } + return Some(tree); + } + } } - /// Checks if this `TokenStream` is empty. #[unstable(feature = "proc_macro", issue = "38356")] - pub fn is_empty(&self) -> bool { - self.0.is_empty() + impl IntoIterator for TokenStream { + type Item = TokenTree; + type IntoIter = IntoIter; + + fn into_iter(self) -> IntoIter { + IntoIter { cursor: self.0.trees(), stack: Vec::new() } + } } } -/// A region of source code, along with macro expansion information. +/// `quote!(..)` accepts arbitrary tokens and expands into a `TokenStream` describing the input. +/// For example, `quote!(a + b)` will produce a expression, that, when evaluated, constructs +/// the `TokenStream` `[Word("a"), Op('+', Alone), Word("b")]`. +/// +/// Unquoting is done with `$`, and works by taking the single next ident as the unquoted term. +/// To quote `$` itself, use `$$`. #[unstable(feature = "proc_macro", issue = "38356")] -#[derive(Copy, Clone, Debug, PartialEq, Eq)] -pub struct Span(syntax_pos::Span); +#[macro_export] +macro_rules! quote { () => {} } -impl Span { - /// A span that resolves at the macro definition site. - #[unstable(feature = "proc_macro", issue = "38356")] - pub fn def_site() -> Span { - ::__internal::with_sess(|(_, mark)| { - let call_site = mark.expn_info().unwrap().call_site; - Span(call_site.with_ctxt(SyntaxContext::empty().apply_mark(mark))) - }) - } -} +#[unstable(feature = "proc_macro_internals", issue = "27812")] +#[doc(hidden)] +mod quote; /// Quote a `Span` into a `TokenStream`. /// This is needed to implement a custom quoter. @@ -196,6 +219,11 @@ pub fn quote_span(span: Span) -> TokenStream { quote::Quote::quote(span) } +/// A region of source code, along with macro expansion information. +#[unstable(feature = "proc_macro", issue = "38356")] +#[derive(Copy, Clone, Debug)] +pub struct Span(syntax_pos::Span); + macro_rules! diagnostic_method { ($name:ident, $level:expr) => ( /// Create a new `Diagnostic` with the given `message` at the span @@ -208,6 +236,15 @@ macro_rules! diagnostic_method { } impl Span { + /// A span that resolves at the macro definition site. + #[unstable(feature = "proc_macro", issue = "38356")] + pub fn def_site() -> Span { + ::__internal::with_sess(|(_, mark)| { + let call_site = mark.expn_info().unwrap().call_site; + Span(call_site.with_ctxt(SyntaxContext::empty().apply_mark(mark))) + }) + } + /// The span of the invocation of the current procedural macro. #[unstable(feature = "proc_macro", issue = "38356")] pub fn call_site() -> Span { @@ -284,6 +321,12 @@ impl Span { other.resolved_at(*self) } + /// Compares to spans to see if they're equal. + #[unstable(feature = "proc_macro", issue = "38356")] + pub fn eq(&self, other: &Span) -> bool { + self.0 == other.0 + } + diagnostic_method!(error, Level::Error); diagnostic_method!(warning, Level::Warning); diagnostic_method!(note, Level::Note); @@ -379,39 +422,97 @@ impl PartialEq for SourceFile { /// A single token or a delimited sequence of token trees (e.g. `[1, (), ..]`). #[unstable(feature = "proc_macro", issue = "38356")] #[derive(Clone, Debug)] -pub struct TokenTree { - /// The `TokenTree`'s span - pub span: Span, - /// Description of the `TokenTree` - pub kind: TokenNode, +pub enum TokenTree { + /// A delimited tokenstream + Group(Group), + /// A unicode identifier + Term(Term), + /// A punctuation character (`+`, `,`, `$`, etc.). + Op(Op), + /// A literal character (`'a'`), string (`"hello"`), number (`2.3`), etc. + Literal(Literal), +} + +impl TokenTree { + /// Returns the span of this token, accessing the `span` method of each of + /// the internal tokens. + #[unstable(feature = "proc_macro", issue = "38356")] + pub fn span(&self) -> Span { + match *self { + TokenTree::Group(ref t) => t.span(), + TokenTree::Term(ref t) => t.span(), + TokenTree::Op(ref t) => t.span(), + TokenTree::Literal(ref t) => t.span(), + } + } + + /// Configures the span for *only this token*. + /// + /// Note that if this token is a `Group` then this method will not configure + /// the span of each of the internal tokens, this will simply delegate to + /// the `set_span` method of each variant. + #[unstable(feature = "proc_macro", issue = "38356")] + pub fn set_span(&mut self, span: Span) { + match *self { + TokenTree::Group(ref mut t) => t.set_span(span), + TokenTree::Term(ref mut t) => t.set_span(span), + TokenTree::Op(ref mut t) => t.set_span(span), + TokenTree::Literal(ref mut t) => t.set_span(span), + } + } +} + +#[unstable(feature = "proc_macro", issue = "38356")] +impl From for TokenTree { + fn from(g: Group) -> TokenTree { + TokenTree::Group(g) + } +} + +#[unstable(feature = "proc_macro", issue = "38356")] +impl From for TokenTree { + fn from(g: Term) -> TokenTree { + TokenTree::Term(g) + } +} + +#[unstable(feature = "proc_macro", issue = "38356")] +impl From for TokenTree { + fn from(g: Op) -> TokenTree { + TokenTree::Op(g) + } } #[unstable(feature = "proc_macro", issue = "38356")] -impl From for TokenTree { - fn from(kind: TokenNode) -> TokenTree { - TokenTree { span: Span::def_site(), kind: kind } +impl From for TokenTree { + fn from(g: Literal) -> TokenTree { + TokenTree::Literal(g) } } #[unstable(feature = "proc_macro", issue = "38356")] impl fmt::Display for TokenTree { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - TokenStream::from(self.clone()).fmt(f) + match *self { + TokenTree::Group(ref t) => t.fmt(f), + TokenTree::Term(ref t) => t.fmt(f), + TokenTree::Op(ref t) => t.fmt(f), + TokenTree::Literal(ref t) => t.fmt(f), + } } } -/// Description of a `TokenTree` +/// A delimited token stream +/// +/// A `Group` internally contains a `TokenStream` which is delimited by a +/// `Delimiter`. Groups represent multiple tokens internally and have a `Span` +/// for the entire stream. #[derive(Clone, Debug)] #[unstable(feature = "proc_macro", issue = "38356")] -pub enum TokenNode { - /// A delimited tokenstream. - Group(Delimiter, TokenStream), - /// A unicode identifier. - Term(Term), - /// A punctuation character (`+`, `,`, `$`, etc.). - Op(char, Spacing), - /// A literal character (`'a'`), string (`"hello"`), or number (`2.3`). - Literal(Literal), +pub struct Group { + delimiter: Delimiter, + stream: TokenStream, + span: Span, } /// Describes how a sequence of token trees is delimited. @@ -428,25 +529,74 @@ pub enum Delimiter { None, } -/// An interned string. -#[derive(Copy, Clone, Debug)] -#[unstable(feature = "proc_macro", issue = "38356")] -pub struct Term(Symbol); +impl Group { + /// Creates a new `group` with the given delimiter and token stream. + /// + /// This constructor will set the span for this group to + /// `Span::call_site()`. To change the span you can use the `set_span` + /// method below. + #[unstable(feature = "proc_macro", issue = "38356")] + pub fn new(delimiter: Delimiter, stream: TokenStream) -> Group { + Group { + delimiter: delimiter, + stream: stream, + span: Span::call_site(), + } + } -impl Term { - /// Intern a string into a `Term`. + /// Returns the delimiter of this `Group` #[unstable(feature = "proc_macro", issue = "38356")] - pub fn intern(string: &str) -> Term { - Term(Symbol::intern(string)) + pub fn delimiter(&self) -> Delimiter { + self.delimiter } - /// Get a reference to the interned string. + /// Returns the `TokenStream` of tokens that are delimited in this `Group`. + /// + /// Note that the returned token stream does not include the delimiter + /// returned above. #[unstable(feature = "proc_macro", issue = "38356")] - pub fn as_str(&self) -> &str { - unsafe { &*(&*self.0.as_str() as *const str) } + pub fn stream(&self) -> TokenStream { + self.stream.clone() + } + + /// Returns the span for the delimiters of this token stream, spanning the + /// entire `Group`. + #[unstable(feature = "proc_macro", issue = "38356")] + pub fn span(&self) -> Span { + self.span + } + + /// Configures the span for this `Group`'s delimiters, but not its internal + /// tokens. + /// + /// This method will **not** set the span of all the internal tokens spanned + /// by this group, but rather it will only set the span of the delimiter + /// tokens at the level of the `Group`. + #[unstable(feature = "proc_macro", issue = "38356")] + pub fn set_span(&mut self, span: Span) { + self.span = span; } } +#[unstable(feature = "proc_macro", issue = "38356")] +impl fmt::Display for Group { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + TokenStream::from(TokenTree::from(self.clone())).fmt(f) + } +} + +/// An `Op` is an operator like `+` or `-`, and only represents one character. +/// +/// Operators like `+=` are represented as two instance of `Op` with different +/// forms of `Spacing` returned. +#[unstable(feature = "proc_macro", issue = "38356")] +#[derive(Copy, Clone, Debug)] +pub struct Op { + op: char, + spacing: Spacing, + span: Span, +} + /// Whether an `Op` is either followed immediately by another `Op` or followed by whitespace. #[derive(Copy, Clone, Debug, PartialEq, Eq)] #[unstable(feature = "proc_macro", issue = "38356")] @@ -457,68 +607,285 @@ pub enum Spacing { Joint, } -/// A literal character (`'a'`), string (`"hello"`), or number (`2.3`). -#[derive(Clone, Debug)] +impl Op { + /// Creates a new `Op` from the given character and spacing. + /// + /// The returned `Op` will have the default span of `Span::call_site()` + /// which can be further configured with the `set_span` method below. + #[unstable(feature = "proc_macro", issue = "38356")] + pub fn new(op: char, spacing: Spacing) -> Op { + Op { + op: op, + spacing: spacing, + span: Span::call_site(), + } + } + + /// Returns the character this operation represents, for example `'+'` + #[unstable(feature = "proc_macro", issue = "38356")] + pub fn op(&self) -> char { + self.op + } + + /// Returns the spacing of this operator, indicating whether it's a joint + /// operator with more operators coming next in the token stream or an + /// `Alone` meaning that the operator has ended. + #[unstable(feature = "proc_macro", issue = "38356")] + pub fn spacing(&self) -> Spacing { + self.spacing + } + + /// Returns the span for this operator character + #[unstable(feature = "proc_macro", issue = "38356")] + pub fn span(&self) -> Span { + self.span + } + + /// Configure the span for this operator's character + #[unstable(feature = "proc_macro", issue = "38356")] + pub fn set_span(&mut self, span: Span) { + self.span = span; + } +} + #[unstable(feature = "proc_macro", issue = "38356")] -pub struct Literal(token::Token); +impl fmt::Display for Op { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + TokenStream::from(TokenTree::from(self.clone())).fmt(f) + } +} +/// An interned string. +#[derive(Copy, Clone, Debug)] #[unstable(feature = "proc_macro", issue = "38356")] -impl fmt::Display for Literal { +pub struct Term { + sym: Symbol, + span: Span, +} + +impl Term { + /// Creates a new `Term` with the given `string` as well as the specified + /// `span`. + /// + /// Note that `span`, currently in rustc, configures the hygiene information + /// for this identifier. As of this time `Span::call_site()` explicitly + /// opts-in to **non-hygienic** information (aka copy/pasted code) while + /// spans like `Span::def_site()` will opt-in to hygienic information, + /// meaning that code at the call site of the macro can't access this + /// identifier. + /// + /// Due to the current importance of hygiene this constructor, unlike other + /// tokens, requires a `Span` to be specified at construction. + #[unstable(feature = "proc_macro", issue = "38356")] + pub fn new(string: &str, span: Span) -> Term { + Term { + sym: Symbol::intern(string), + span, + } + } + + /// Get a reference to the interned string. + #[unstable(feature = "proc_macro", issue = "38356")] + pub fn as_str(&self) -> &str { + unsafe { &*(&*self.sym.as_str() as *const str) } + } + + /// Returns the span of this `Term`, encompassing the entire string returned + /// by `as_str`. + #[unstable(feature = "proc_macro", issue = "38356")] + pub fn span(&self) -> Span { + self.span + } + + /// Configures the span of this `Term`, possibly changing hygiene + /// information. + #[unstable(feature = "proc_macro", issue = "38356")] + pub fn set_span(&mut self, span: Span) { + self.span = span; + } +} + +#[unstable(feature = "proc_macro", issue = "38356")] +impl fmt::Display for Term { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - TokenTree { kind: TokenNode::Literal(self.clone()), span: Span(DUMMY_SP) }.fmt(f) + self.as_str().fmt(f) } } -macro_rules! int_literals { - ($($int_kind:ident),*) => {$( - /// Integer literal. +/// A literal character (`'a'`), string (`"hello"`), a number (`2.3`), etc. +#[derive(Clone, Debug)] +#[unstable(feature = "proc_macro", issue = "38356")] +pub struct Literal { + token: token::Token, + span: Span, +} + +macro_rules! suffixed_int_literals { + ($($name:ident => $kind:ident,)*) => ($( + /// Creates a new suffixed integer literal with the specified value. + /// + /// This function will create an integer like `1u32` where the integer + /// value specified is the first part of the token and the integral is + /// also suffixed at the end. + /// + /// Literals created through this method have the `Span::call_site()` + /// span by default, which can be configured with the `set_span` method + /// below. #[unstable(feature = "proc_macro", issue = "38356")] - pub fn $int_kind(n: $int_kind) -> Literal { - Literal::typed_integer(n as i128, stringify!($int_kind)) + pub fn $name(n: $kind) -> Literal { + let lit = token::Lit::Integer(Symbol::intern(&n.to_string())); + let ty = Some(Symbol::intern(stringify!($kind))); + Literal { + token: token::Literal(lit, ty), + span: Span::call_site(), + } + } + )*) +} + +macro_rules! unsuffixed_int_literals { + ($($name:ident => $kind:ident,)*) => ($( + /// Creates a new unsuffixed integer literal with the specified value. + /// + /// This function will create an integer like `1` where the integer + /// value specified is the first part of the token. No suffix is + /// specified on this token, meaning that invocations like + /// `Literal::i8_unsuffixed(1)` are equivalent to + /// `Literal::u32_unsuffixed(1)`. + /// + /// Literals created through this method have the `Span::call_site()` + /// span by default, which can be configured with the `set_span` method + /// below. + #[unstable(feature = "proc_macro", issue = "38356")] + pub fn $name(n: $kind) -> Literal { + let lit = token::Lit::Integer(Symbol::intern(&n.to_string())); + Literal { + token: token::Literal(lit, None), + span: Span::call_site(), + } } - )*} + )*) } impl Literal { - /// Integer literal - #[unstable(feature = "proc_macro", issue = "38356")] - pub fn integer(n: i128) -> Literal { - Literal(token::Literal(token::Lit::Integer(Symbol::intern(&n.to_string())), None)) + suffixed_int_literals! { + u8_suffixed => u8, + u16_suffixed => u16, + u32_suffixed => u32, + u64_suffixed => u64, + u128_suffixed => u128, + usize_suffixed => usize, + i8_suffixed => i8, + i16_suffixed => i16, + i32_suffixed => i32, + i64_suffixed => i64, + i128_suffixed => i128, + isize_suffixed => isize, } - int_literals!(u8, i8, u16, i16, u32, i32, u64, i64, usize, isize); - fn typed_integer(n: i128, kind: &'static str) -> Literal { - Literal(token::Literal(token::Lit::Integer(Symbol::intern(&n.to_string())), - Some(Symbol::intern(kind)))) + unsuffixed_int_literals! { + u8_unsuffixed => u8, + u16_unsuffixed => u16, + u32_unsuffixed => u32, + u64_unsuffixed => u64, + u128_unsuffixed => u128, + usize_unsuffixed => usize, + i8_unsuffixed => i8, + i16_unsuffixed => i16, + i32_unsuffixed => i32, + i64_unsuffixed => i64, + i128_unsuffixed => i128, + isize_unsuffixed => isize, } - /// Floating point literal. + /// Creates a new unsuffixed floating-point literal. + /// + /// This constructor is similar to those like `Literal::i8_unsuffixed` where + /// the float's value is emitted directly into the token but no suffix is + /// used, so it may be inferred to be a `f64` later in the compiler. + /// + /// # Panics + /// + /// This function requires that the specified float is finite, for + /// example if it is infinity or NaN this function will panic. #[unstable(feature = "proc_macro", issue = "38356")] - pub fn float(n: f64) -> Literal { + pub fn f32_unsuffixed(n: f32) -> Literal { if !n.is_finite() { panic!("Invalid float literal {}", n); } - Literal(token::Literal(token::Lit::Float(Symbol::intern(&n.to_string())), None)) + let lit = token::Lit::Float(Symbol::intern(&n.to_string())); + Literal { + token: token::Literal(lit, None), + span: Span::call_site(), + } } - /// Floating point literal. + /// Creates a new suffixed floating-point literal. + /// + /// This consturctor will create a literal like `1.0f32` where the value + /// specified is the preceding part of the token and `f32` is the suffix of + /// the token. This token will always be inferred to be an `f32` in the + /// compiler. + /// + /// # Panics + /// + /// This function requires that the specified float is finite, for + /// example if it is infinity or NaN this function will panic. #[unstable(feature = "proc_macro", issue = "38356")] - pub fn f32(n: f32) -> Literal { + pub fn f32_suffixed(n: f32) -> Literal { if !n.is_finite() { - panic!("Invalid f32 literal {}", n); + panic!("Invalid float literal {}", n); + } + let lit = token::Lit::Float(Symbol::intern(&n.to_string())); + Literal { + token: token::Literal(lit, Some(Symbol::intern("f32"))), + span: Span::call_site(), } - Literal(token::Literal(token::Lit::Float(Symbol::intern(&n.to_string())), - Some(Symbol::intern("f32")))) } - /// Floating point literal. + /// Creates a new unsuffixed floating-point literal. + /// + /// This constructor is similar to those like `Literal::i8_unsuffixed` where + /// the float's value is emitted directly into the token but no suffix is + /// used, so it may be inferred to be a `f64` later in the compiler. + /// + /// # Panics + /// + /// This function requires that the specified float is finite, for + /// example if it is infinity or NaN this function will panic. #[unstable(feature = "proc_macro", issue = "38356")] - pub fn f64(n: f64) -> Literal { + pub fn f64_unsuffixed(n: f64) -> Literal { if !n.is_finite() { - panic!("Invalid f64 literal {}", n); + panic!("Invalid float literal {}", n); + } + let lit = token::Lit::Float(Symbol::intern(&n.to_string())); + Literal { + token: token::Literal(lit, None), + span: Span::call_site(), + } + } + + /// Creates a new suffixed floating-point literal. + /// + /// This consturctor will create a literal like `1.0f64` where the value + /// specified is the preceding part of the token and `f64` is the suffix of + /// the token. This token will always be inferred to be an `f64` in the + /// compiler. + /// + /// # Panics + /// + /// This function requires that the specified float is finite, for + /// example if it is infinity or NaN this function will panic. + #[unstable(feature = "proc_macro", issue = "38356")] + pub fn f64_suffixed(n: f64) -> Literal { + if !n.is_finite() { + panic!("Invalid float literal {}", n); + } + let lit = token::Lit::Float(Symbol::intern(&n.to_string())); + Literal { + token: token::Literal(lit, Some(Symbol::intern("f64"))), + span: Span::call_site(), } - Literal(token::Literal(token::Lit::Float(Symbol::intern(&n.to_string())), - Some(Symbol::intern("f64")))) } /// String literal. @@ -528,7 +895,10 @@ impl Literal { for ch in string.chars() { escaped.extend(ch.escape_debug()); } - Literal(token::Literal(token::Lit::Str_(Symbol::intern(&escaped)), None)) + Literal { + token: token::Literal(token::Lit::Str_(Symbol::intern(&escaped)), None), + span: Span::call_site(), + } } /// Character literal. @@ -536,7 +906,10 @@ impl Literal { pub fn character(ch: char) -> Literal { let mut escaped = String::new(); escaped.extend(ch.escape_unicode()); - Literal(token::Literal(token::Lit::Char(Symbol::intern(&escaped)), None)) + Literal { + token: token::Literal(token::Lit::Char(Symbol::intern(&escaped)), None), + span: Span::call_site(), + } } /// Byte string literal. @@ -544,36 +917,29 @@ impl Literal { pub fn byte_string(bytes: &[u8]) -> Literal { let string = bytes.iter().cloned().flat_map(ascii::escape_default) .map(Into::::into).collect::(); - Literal(token::Literal(token::Lit::ByteStr(Symbol::intern(&string)), None)) + Literal { + token: token::Literal(token::Lit::ByteStr(Symbol::intern(&string)), None), + span: Span::call_site(), + } } -} -/// An iterator over `TokenTree`s. -#[derive(Clone)] -#[unstable(feature = "proc_macro", issue = "38356")] -pub struct TokenTreeIter { - cursor: tokenstream::Cursor, - stack: Vec, + /// Returns the span encompassing this literal. + #[unstable(feature = "proc_macro", issue = "38356")] + pub fn span(&self) -> Span { + self.span + } + + /// Configures the span associated for this literal. + #[unstable(feature = "proc_macro", issue = "38356")] + pub fn set_span(&mut self, span: Span) { + self.span = span; + } } #[unstable(feature = "proc_macro", issue = "38356")] -impl Iterator for TokenTreeIter { - type Item = TokenTree; - - fn next(&mut self) -> Option { - loop { - let tree = self.stack.pop().or_else(|| { - let next = self.cursor.next_as_stream()?; - Some(TokenTree::from_internal(next, &mut self.stack)) - })?; - if tree.span.0 == DUMMY_SP { - if let TokenNode::Group(Delimiter::None, stream) = tree.kind { - self.cursor.insert(stream.0); - continue - } - } - return Some(tree); - } +impl fmt::Display for Literal { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + TokenStream::from(TokenTree::from(self.clone())).fmt(f) } } @@ -607,31 +973,34 @@ impl TokenTree { tokenstream::TokenTree::Token(span, token) => (span, token), tokenstream::TokenTree::Delimited(span, delimed) => { let delimiter = Delimiter::from_internal(delimed.delim); - return TokenTree { - span: Span(span), - kind: TokenNode::Group(delimiter, TokenStream(delimed.tts.into())), - }; + let mut g = Group::new(delimiter, TokenStream(delimed.tts.into())); + g.set_span(Span(span)); + return g.into() } }; let op_kind = if is_joint { Spacing::Joint } else { Spacing::Alone }; macro_rules! tt { - ($e:expr) => (TokenTree { span: Span(span), kind: $e }) + ($e:expr) => ({ + let mut x = TokenTree::from($e); + x.set_span(Span(span)); + x + }) } macro_rules! op { - ($a:expr) => (TokenNode::Op($a, op_kind)); + ($a:expr) => (tt!(Op::new($a, op_kind))); ($a:expr, $b:expr) => ({ - stack.push(tt!(TokenNode::Op($b, op_kind).into())); - TokenNode::Op($a, Spacing::Joint) + stack.push(tt!(Op::new($b, op_kind))); + tt!(Op::new($a, Spacing::Joint)) }); ($a:expr, $b:expr, $c:expr) => ({ - stack.push(tt!(TokenNode::Op($c, op_kind))); - stack.push(tt!(TokenNode::Op($b, Spacing::Joint))); - TokenNode::Op($a, Spacing::Joint) + stack.push(tt!(Op::new($c, op_kind))); + stack.push(tt!(Op::new($b, Spacing::Joint))); + tt!(Op::new($a, Spacing::Joint)) }) } - let kind = match token { + match token { Eq => op!('='), Lt => op!('<'), Le => op!('<', '='), @@ -679,80 +1048,88 @@ impl TokenTree { Dollar => op!('$'), Question => op!('?'), - Ident(ident, false) | Lifetime(ident) => TokenNode::Term(Term(ident.name)), - Ident(ident, true) => TokenNode::Term(Term(Symbol::intern(&format!("r#{}", ident)))), - Literal(..) => TokenNode::Literal(self::Literal(token)), + Ident(ident, false) | Lifetime(ident) => { + tt!(Term::new(&ident.name.as_str(), Span(span))) + } + Ident(ident, true) => { + tt!(Term::new(&format!("r#{}", ident), Span(span))) + } + Literal(..) => tt!(self::Literal { token, span: Span(span) }), DocComment(c) => { let stream = vec![ - tt!(TokenNode::Term(Term::intern("doc"))), - tt!(op!('=')), - tt!(TokenNode::Literal(self::Literal(Literal(Lit::Str_(c), None)))), + tt!(Term::new("doc", Span(span))), + tt!(Op::new('=', Spacing::Alone)), + tt!(self::Literal::string(&c.as_str())), ].into_iter().collect(); - stack.push(tt!(TokenNode::Group(Delimiter::Bracket, stream))); - op!('#') + stack.push(tt!(Group::new(Delimiter::Bracket, stream))); + tt!(Op::new('#', Spacing::Alone)) } Interpolated(_) => { __internal::with_sess(|(sess, _)| { let tts = token.interpolated_to_tokenstream(sess, span); - TokenNode::Group(Delimiter::None, TokenStream(tts)) + tt!(Group::new(Delimiter::None, TokenStream(tts))) }) } DotEq => op!('.', '='), OpenDelim(..) | CloseDelim(..) => unreachable!(), Whitespace | Comment | Shebang(..) | Eof => unreachable!(), - }; - - TokenTree { span: Span(span), kind: kind } + } } fn to_internal(self) -> tokenstream::TokenStream { use syntax::parse::token::*; use syntax::tokenstream::{TokenTree, Delimited}; - let (op, kind) = match self.kind { - TokenNode::Op(op, kind) => (op, kind), - TokenNode::Group(delimiter, tokens) => { - return TokenTree::Delimited(self.span.0, Delimited { - delim: delimiter.to_internal(), - tts: tokens.0.into(), + let (op, kind, span) = match self { + self::TokenTree::Op(tt) => (tt.op(), tt.spacing(), tt.span()), + self::TokenTree::Group(tt) => { + return TokenTree::Delimited(tt.span.0, Delimited { + delim: tt.delimiter.to_internal(), + tts: tt.stream.0.into(), }).into(); }, - TokenNode::Term(symbol) => { - let ident = ast::Ident { name: symbol.0, ctxt: self.span.0.ctxt() }; - let sym_str = symbol.0.as_str(); + self::TokenTree::Term(tt) => { + let ident = ast::Ident { name: tt.sym, ctxt: tt.span.0.ctxt() }; + let sym_str = tt.sym.as_str(); let token = if sym_str.starts_with("'") { Lifetime(ident) } else if sym_str.starts_with("r#") { let name = Symbol::intern(&sym_str[2..]); - let ident = ast::Ident { name, ctxt: self.span.0.ctxt() }; + let ident = ast::Ident { name, ctxt: tt.span.0.ctxt() }; Ident(ident, true) } else { Ident(ident, false) }; - return TokenTree::Token(self.span.0, token).into(); + return TokenTree::Token(tt.span.0, token).into(); } - TokenNode::Literal(self::Literal(Literal(Lit::Integer(ref a), b))) + self::TokenTree::Literal(self::Literal { + token: Literal(Lit::Integer(ref a), b), + span, + }) if a.as_str().starts_with("-") => { let minus = BinOp(BinOpToken::Minus); let integer = Symbol::intern(&a.as_str()[1..]); let integer = Literal(Lit::Integer(integer), b); - let a = TokenTree::Token(self.span.0, minus); - let b = TokenTree::Token(self.span.0, integer); + let a = TokenTree::Token(span.0, minus); + let b = TokenTree::Token(span.0, integer); return vec![a, b].into_iter().collect() } - TokenNode::Literal(self::Literal(Literal(Lit::Float(ref a), b))) + self::TokenTree::Literal(self::Literal { + token: Literal(Lit::Float(ref a), b), + span, + }) if a.as_str().starts_with("-") => { let minus = BinOp(BinOpToken::Minus); let float = Symbol::intern(&a.as_str()[1..]); let float = Literal(Lit::Float(float), b); - let a = TokenTree::Token(self.span.0, minus); - let b = TokenTree::Token(self.span.0, float); + let a = TokenTree::Token(span.0, minus); + let b = TokenTree::Token(span.0, float); return vec![a, b].into_iter().collect() } - TokenNode::Literal(token) => { - return TokenTree::Token(self.span.0, token.0).into() + self::TokenTree::Literal(tt) => { + return TokenTree::Token(tt.span.0, tt.token).into() } }; @@ -781,7 +1158,7 @@ impl TokenTree { _ => panic!("unsupported character {}", op), }; - let tree = TokenTree::Token(self.span.0, token); + let tree = TokenTree::Token(span.0, token); match kind { Spacing::Alone => tree.into(), Spacing::Joint => tree.joint(), diff --git a/src/libproc_macro/quote.rs b/src/libproc_macro/quote.rs index 8b5add1a0f0d7..cc8575b88be97 100644 --- a/src/libproc_macro/quote.rs +++ b/src/libproc_macro/quote.rs @@ -14,7 +14,7 @@ //! This quasiquoter uses macros 2.0 hygiene to reliably access //! items from `proc_macro`, to build a `proc_macro::TokenStream`. -use {Delimiter, Literal, Spacing, Span, Term, TokenNode, TokenStream, TokenTree}; +use {Delimiter, Literal, Spacing, Span, Term, Op, Group, TokenStream, TokenTree}; use syntax::ext::base::{ExtCtxt, ProcMacro}; use syntax::parse::token; @@ -23,47 +23,59 @@ use syntax::tokenstream; pub struct Quoter; pub fn unquote + Clone>(tokens: &T) -> TokenStream { - T::into(tokens.clone()) + tokens.clone().into() } pub trait Quote { fn quote(self) -> TokenStream; } +macro_rules! tt2ts { + ($e:expr) => (TokenStream::from(TokenTree::from($e))) +} + macro_rules! quote_tok { - (,) => { TokenNode::Op(',', Spacing::Alone) }; - (.) => { TokenNode::Op('.', Spacing::Alone) }; - (:) => { TokenNode::Op(':', Spacing::Alone) }; + (,) => { tt2ts!(Op::new(',', Spacing::Alone)) }; + (.) => { tt2ts!(Op::new('.', Spacing::Alone)) }; + (:) => { tt2ts!(Op::new(':', Spacing::Alone)) }; + (|) => { tt2ts!(Op::new('|', Spacing::Alone)) }; (::) => { [ - TokenNode::Op(':', Spacing::Joint), - TokenNode::Op(':', Spacing::Alone) - ].iter().cloned().collect::() + TokenTree::from(Op::new(':', Spacing::Joint)), + TokenTree::from(Op::new(':', Spacing::Alone)), + ].iter() + .cloned() + .map(|mut x| { + x.set_span(Span::def_site()); + x + }) + .collect::() }; - (!) => { TokenNode::Op('!', Spacing::Alone) }; - (<) => { TokenNode::Op('<', Spacing::Alone) }; - (>) => { TokenNode::Op('>', Spacing::Alone) }; - (_) => { TokenNode::Op('_', Spacing::Alone) }; - (0) => { TokenNode::Literal(::Literal::integer(0)) }; - (&) => { TokenNode::Op('&', Spacing::Alone) }; - ($i:ident) => { TokenNode::Term(Term::intern(stringify!($i))) }; + (!) => { tt2ts!(Op::new('!', Spacing::Alone)) }; + (<) => { tt2ts!(Op::new('<', Spacing::Alone)) }; + (>) => { tt2ts!(Op::new('>', Spacing::Alone)) }; + (_) => { tt2ts!(Op::new('_', Spacing::Alone)) }; + (0) => { tt2ts!(Literal::i8_unsuffixed(0)) }; + (&) => { tt2ts!(Op::new('&', Spacing::Alone)) }; + ($i:ident) => { tt2ts!(Term::new(stringify!($i), Span::def_site())) }; } macro_rules! quote_tree { ((unquote $($t:tt)*)) => { $($t)* }; ((quote $($t:tt)*)) => { ($($t)*).quote() }; - (($($t:tt)*)) => { TokenNode::Group(Delimiter::Parenthesis, quote!($($t)*)) }; - ([$($t:tt)*]) => { TokenNode::Group(Delimiter::Bracket, quote!($($t)*)) }; - ({$($t:tt)*}) => { TokenNode::Group(Delimiter::Brace, quote!($($t)*)) }; + (($($t:tt)*)) => { tt2ts!(Group::new(Delimiter::Parenthesis, quote!($($t)*))) }; + ([$($t:tt)*]) => { tt2ts!(Group::new(Delimiter::Bracket, quote!($($t)*))) }; + ({$($t:tt)*}) => { tt2ts!(Group::new(Delimiter::Brace, quote!($($t)*))) }; ($t:tt) => { quote_tok!($t) }; } macro_rules! quote { () => { TokenStream::empty() }; ($($t:tt)*) => { - [ - $(TokenStream::from(quote_tree!($t)),)* - ].iter().cloned().collect::() + [$(quote_tree!($t),)*].iter() + .cloned() + .flat_map(|x| x.into_iter()) + .collect::() }; } @@ -97,72 +109,81 @@ impl Quote for TokenStream { let tokens = self.into_iter().filter_map(|tree| { if after_dollar { after_dollar = false; - match tree.kind { - TokenNode::Term(_) => { + match tree { + TokenTree::Term(_) => { + let tree = TokenStream::from(tree); return Some(quote!(::__internal::unquote(&(unquote tree)),)); } - TokenNode::Op('$', _) => {} + TokenTree::Op(ref tt) if tt.op() == '$' => {} _ => panic!("`$` must be followed by an ident or `$` in `quote!`"), } - } else if let TokenNode::Op('$', _) = tree.kind { - after_dollar = true; - return None; + } else if let TokenTree::Op(tt) = tree { + if tt.op() == '$' { + after_dollar = true; + return None; + } } Some(quote!(::TokenStream::from((quote tree)),)) - }).collect::(); + }).flat_map(|t| t.into_iter()).collect::(); if after_dollar { panic!("unexpected trailing `$` in `quote!`"); } - quote!([(unquote tokens)].iter().cloned().collect::<::TokenStream>()) + quote!( + [(unquote tokens)].iter() + .cloned() + .flat_map(|x| x.into_iter()) + .collect::<::TokenStream>() + ) } } impl Quote for TokenTree { fn quote(self) -> TokenStream { - quote!(::TokenTree { span: (quote self.span), kind: (quote self.kind) }) + match self { + TokenTree::Op(tt) => quote!(::TokenTree::Op( (quote tt) )), + TokenTree::Group(tt) => quote!(::TokenTree::Group( (quote tt) )), + TokenTree::Term(tt) => quote!(::TokenTree::Term( (quote tt) )), + TokenTree::Literal(tt) => quote!(::TokenTree::Literal( (quote tt) )), + } } } -impl Quote for TokenNode { +impl Quote for char { fn quote(self) -> TokenStream { - macro_rules! gen_match { - ($($i:ident($($arg:ident),+)),*) => { - match self { - $(TokenNode::$i($($arg),+) => quote! { - ::TokenNode::$i($((quote $arg)),+) - },)* - } - } - } + TokenTree::from(Literal::character(self)).into() + } +} - gen_match! { Op(op, kind), Group(delim, tokens), Term(term), Literal(lit) } +impl<'a> Quote for &'a str { + fn quote(self) -> TokenStream { + TokenTree::from(Literal::string(self)).into() } } -impl Quote for char { +impl Quote for usize { fn quote(self) -> TokenStream { - TokenNode::Literal(Literal::character(self)).into() + TokenTree::from(Literal::usize_unsuffixed(self)).into() } } -impl<'a> Quote for &'a str { +impl Quote for Group { fn quote(self) -> TokenStream { - TokenNode::Literal(Literal::string(self)).into() + quote!(::Group::new((quote self.delimiter()), (quote self.stream()))) } } -impl Quote for usize { +impl Quote for Op { fn quote(self) -> TokenStream { - TokenNode::Literal(Literal::integer(self as i128)).into() + quote!(::Op::new((quote self.op()), (quote self.spacing()))) } } impl Quote for Term { fn quote(self) -> TokenStream { - quote!(::Term::intern((quote self.as_str()))) + quote!(::Term::new((quote self.as_str()), (quote self.span()))) } } @@ -182,14 +203,20 @@ macro_rules! literals { impl LiteralKind { pub fn with_contents_and_suffix(self, contents: Term, suffix: Option) -> Literal { - let contents = contents.0; - let suffix = suffix.map(|t| t.0); + let sym = contents.sym; + let suffix = suffix.map(|t| t.sym); match self { $(LiteralKind::$i => { - Literal(token::Literal(token::Lit::$i(contents), suffix)) + Literal { + token: token::Literal(token::Lit::$i(sym), suffix), + span: contents.span, + } })* $(LiteralKind::$raw(n) => { - Literal(token::Literal(token::Lit::$raw(contents, n), suffix)) + Literal { + token: token::Literal(token::Lit::$raw(sym, n), suffix), + span: contents.span, + } })* } } @@ -197,16 +224,17 @@ macro_rules! literals { impl Literal { fn kind_contents_and_suffix(self) -> (LiteralKind, Term, Option) { - let (lit, suffix) = match self.0 { + let (lit, suffix) = match self.token { token::Literal(lit, suffix) => (lit, suffix), - _ => panic!("unsupported literal {:?}", self.0), + _ => panic!("unsupported literal {:?}", self.token), }; let (kind, contents) = match lit { $(token::Lit::$i(contents) => (LiteralKind::$i, contents),)* $(token::Lit::$raw(contents, n) => (LiteralKind::$raw(n), contents),)* }; - (kind, Term(contents), suffix.map(Term)) + let suffix = suffix.map(|sym| Term::new(&sym.as_str(), self.span())); + (kind, Term::new(&contents.as_str(), self.span()), suffix) } } diff --git a/src/test/compile-fail-fulldeps/proc-macro/auxiliary/attributes-included.rs b/src/test/compile-fail-fulldeps/proc-macro/auxiliary/attributes-included.rs index d725adfec7544..bbfec5815ba5c 100644 --- a/src/test/compile-fail-fulldeps/proc-macro/auxiliary/attributes-included.rs +++ b/src/test/compile-fail-fulldeps/proc-macro/auxiliary/attributes-included.rs @@ -16,7 +16,7 @@ extern crate proc_macro; -use proc_macro::{TokenStream, TokenTree, TokenNode, Delimiter, Literal, Spacing}; +use proc_macro::{TokenStream, TokenTree, Delimiter, Literal, Spacing, Group}; #[proc_macro_attribute] pub fn foo(attr: TokenStream, input: TokenStream) -> TokenStream { @@ -52,24 +52,30 @@ pub fn bar(attr: TokenStream, input: TokenStream) -> TokenStream { } fn assert_inline(slice: &mut &[TokenTree]) { - match slice[0].kind { - TokenNode::Op('#', _) => {} + match &slice[0] { + TokenTree::Op(tt) => assert_eq!(tt.op(), '#'), _ => panic!("expected '#' char"), } - match slice[1].kind { - TokenNode::Group(Delimiter::Bracket, _) => {} + match &slice[1] { + TokenTree::Group(tt) => assert_eq!(tt.delimiter(), Delimiter::Bracket), _ => panic!("expected brackets"), } *slice = &slice[2..]; } fn assert_doc(slice: &mut &[TokenTree]) { - match slice[0].kind { - TokenNode::Op('#', Spacing::Alone) => {} + match &slice[0] { + TokenTree::Op(tt) => { + assert_eq!(tt.op(), '#'); + assert_eq!(tt.spacing(), Spacing::Alone); + } _ => panic!("expected #"), } - let inner = match slice[1].kind { - TokenNode::Group(Delimiter::Bracket, ref s) => s.clone(), + let inner = match &slice[1] { + TokenTree::Group(tt) => { + assert_eq!(tt.delimiter(), Delimiter::Bracket); + tt.stream() + } _ => panic!("expected brackets"), }; let tokens = inner.into_iter().collect::>(); @@ -79,16 +85,19 @@ fn assert_doc(slice: &mut &[TokenTree]) { panic!("expected three tokens in doc") } - match tokens[0].kind { - TokenNode::Term(ref t) => assert_eq!("doc", t.as_str()), + match &tokens[0] { + TokenTree::Term(tt) => assert_eq!("doc", tt.as_str()), _ => panic!("expected `doc`"), } - match tokens[1].kind { - TokenNode::Op('=', Spacing::Alone) => {} + match &tokens[1] { + TokenTree::Op(tt) => { + assert_eq!(tt.op(), '='); + assert_eq!(tt.spacing(), Spacing::Alone); + } _ => panic!("expected equals"), } - match tokens[2].kind { - TokenNode::Literal(_) => {} + match tokens[2] { + TokenTree::Literal(_) => {} _ => panic!("expected literal"), } @@ -96,32 +105,35 @@ fn assert_doc(slice: &mut &[TokenTree]) { } fn assert_invoc(slice: &mut &[TokenTree]) { - match slice[0].kind { - TokenNode::Op('#', _) => {} + match &slice[0] { + TokenTree::Op(tt) => assert_eq!(tt.op(), '#'), _ => panic!("expected '#' char"), } - match slice[1].kind { - TokenNode::Group(Delimiter::Bracket, _) => {} + match &slice[1] { + TokenTree::Group(tt) => assert_eq!(tt.delimiter(), Delimiter::Bracket), _ => panic!("expected brackets"), } *slice = &slice[2..]; } fn assert_foo(slice: &mut &[TokenTree]) { - match slice[0].kind { - TokenNode::Term(ref name) => assert_eq!(name.as_str(), "fn"), + match &slice[0] { + TokenTree::Term(tt) => assert_eq!(tt.as_str(), "fn"), _ => panic!("expected fn"), } - match slice[1].kind { - TokenNode::Term(ref name) => assert_eq!(name.as_str(), "foo"), + match &slice[1] { + TokenTree::Term(tt) => assert_eq!(tt.as_str(), "foo"), _ => panic!("expected foo"), } - match slice[2].kind { - TokenNode::Group(Delimiter::Parenthesis, ref s) => assert!(s.is_empty()), + match &slice[2] { + TokenTree::Group(tt) => { + assert_eq!(tt.delimiter(), Delimiter::Parenthesis); + assert!(tt.stream().is_empty()); + } _ => panic!("expected parens"), } - match slice[3].kind { - TokenNode::Group(Delimiter::Brace, _) => {} + match &slice[3] { + TokenTree::Group(tt) => assert_eq!(tt.delimiter(), Delimiter::Brace), _ => panic!("expected braces"), } *slice = &slice[4..]; @@ -132,22 +144,17 @@ fn fold_stream(input: TokenStream) -> TokenStream { } fn fold_tree(input: TokenTree) -> TokenTree { - TokenTree { - span: input.span, - kind: fold_node(input.kind), - } -} - -fn fold_node(input: TokenNode) -> TokenNode { match input { - TokenNode::Group(a, b) => TokenNode::Group(a, fold_stream(b)), - TokenNode::Op(a, b) => TokenNode::Op(a, b), - TokenNode::Term(a) => TokenNode::Term(a), - TokenNode::Literal(a) => { + TokenTree::Group(b) => { + TokenTree::Group(Group::new(b.delimiter(), fold_stream(b.stream()))) + } + TokenTree::Op(b) => TokenTree::Op(b), + TokenTree::Term(a) => TokenTree::Term(a), + TokenTree::Literal(a) => { if a.to_string() != "\"foo\"" { - TokenNode::Literal(a) + TokenTree::Literal(a) } else { - TokenNode::Literal(Literal::integer(3)) + TokenTree::Literal(Literal::i32_unsuffixed(3)) } } } diff --git a/src/test/run-pass-fulldeps/auxiliary/cond_plugin.rs b/src/test/run-pass-fulldeps/auxiliary/cond_plugin.rs index e2c68a626f91e..281ee70815e11 100644 --- a/src/test/run-pass-fulldeps/auxiliary/cond_plugin.rs +++ b/src/test/run-pass-fulldeps/auxiliary/cond_plugin.rs @@ -15,15 +15,15 @@ extern crate proc_macro; -use proc_macro::{TokenStream, TokenNode, quote}; +use proc_macro::*; #[proc_macro] pub fn cond(input: TokenStream) -> TokenStream { let mut conds = Vec::new(); let mut input = input.into_iter().peekable(); while let Some(tree) = input.next() { - let cond = match tree.kind { - TokenNode::Group(_, cond) => cond, + let cond = match tree { + TokenTree::Group(tt) => tt.stream(), _ => panic!("Invalid input"), }; let mut cond_trees = cond.clone().into_iter(); @@ -32,8 +32,8 @@ pub fn cond(input: TokenStream) -> TokenStream { if rhs.is_empty() { panic!("Invalid macro usage in cond: {}", cond); } - let is_else = match test.kind { - TokenNode::Term(word) => word.as_str() == "else", + let is_else = match test { + TokenTree::Term(word) => word.as_str() == "else", _ => false, }; conds.push(if is_else || input.peek().is_none() { @@ -43,5 +43,5 @@ pub fn cond(input: TokenStream) -> TokenStream { }); } - conds.into_iter().collect() + conds.into_iter().flat_map(|x| x.into_iter()).collect() } diff --git a/src/test/run-pass-fulldeps/auxiliary/proc_macro_def.rs b/src/test/run-pass-fulldeps/auxiliary/proc_macro_def.rs index 1b47043884844..d3670ae66feed 100644 --- a/src/test/run-pass-fulldeps/auxiliary/proc_macro_def.rs +++ b/src/test/run-pass-fulldeps/auxiliary/proc_macro_def.rs @@ -15,7 +15,7 @@ extern crate proc_macro; -use proc_macro::{TokenStream, quote}; +use proc_macro::*; #[proc_macro_attribute] pub fn attr_tru(_attr: TokenStream, item: TokenStream) -> TokenStream { diff --git a/src/test/run-pass-fulldeps/proc-macro/auxiliary/count_compound_ops.rs b/src/test/run-pass-fulldeps/proc-macro/auxiliary/count_compound_ops.rs index ec2ff0d1e2b8c..063d8dc40536d 100644 --- a/src/test/run-pass-fulldeps/proc-macro/auxiliary/count_compound_ops.rs +++ b/src/test/run-pass-fulldeps/proc-macro/auxiliary/count_compound_ops.rs @@ -15,20 +15,25 @@ extern crate proc_macro; -use proc_macro::{TokenStream, TokenNode, Spacing, Literal, quote}; +use proc_macro::{TokenStream, TokenTree, Spacing, Literal, quote}; #[proc_macro] pub fn count_compound_ops(input: TokenStream) -> TokenStream { assert_eq!(count_compound_ops_helper(quote!(++ (&&) 4@a)), 3); - TokenNode::Literal(Literal::u32(count_compound_ops_helper(input))).into() + let l = Literal::u32_suffixed(count_compound_ops_helper(input)); + TokenTree::from(l).into() } fn count_compound_ops_helper(input: TokenStream) -> u32 { let mut count = 0; for token in input { - match token.kind { - TokenNode::Op(c, Spacing::Alone) => count += 1, - TokenNode::Group(_, tokens) => count += count_compound_ops_helper(tokens), + match &token { + TokenTree::Op(tt) if tt.spacing() == Spacing::Alone => { + count += 1; + } + TokenTree::Group(tt) => { + count += count_compound_ops_helper(tt.stream()); + } _ => {} } } diff --git a/src/test/run-pass-fulldeps/proc-macro/auxiliary/negative-token.rs b/src/test/run-pass-fulldeps/proc-macro/auxiliary/negative-token.rs index e5ebb7c2e41b3..e76e4d585f497 100644 --- a/src/test/run-pass-fulldeps/proc-macro/auxiliary/negative-token.rs +++ b/src/test/run-pass-fulldeps/proc-macro/auxiliary/negative-token.rs @@ -19,16 +19,10 @@ use proc_macro::*; #[proc_macro] pub fn neg_one(_input: TokenStream) -> TokenStream { - TokenTree { - span: Span::call_site(), - kind: TokenNode::Literal(Literal::i32(-1)), - }.into() + TokenTree::Literal(Literal::i32_suffixed(-1)).into() } #[proc_macro] pub fn neg_one_float(_input: TokenStream) -> TokenStream { - TokenTree { - span: Span::call_site(), - kind: TokenNode::Literal(Literal::f32(-1.0)), - }.into() + TokenTree::Literal(Literal::f32_suffixed(-1.0)).into() } diff --git a/src/test/run-pass-fulldeps/proc-macro/auxiliary/span-api-tests.rs b/src/test/run-pass-fulldeps/proc-macro/auxiliary/span-api-tests.rs index ce6ffcc3cb03d..6ab9d6d0b8a7c 100644 --- a/src/test/run-pass-fulldeps/proc-macro/auxiliary/span-api-tests.rs +++ b/src/test/run-pass-fulldeps/proc-macro/auxiliary/span-api-tests.rs @@ -27,7 +27,7 @@ pub fn reemit(input: TokenStream) -> TokenStream { #[proc_macro] pub fn assert_fake_source_file(input: TokenStream) -> TokenStream { for tk in input { - let source_file = tk.span.source_file(); + let source_file = tk.span().source_file(); assert!(!source_file.is_real(), "Source file is real: {:?}", source_file); } @@ -37,7 +37,7 @@ pub fn assert_fake_source_file(input: TokenStream) -> TokenStream { #[proc_macro] pub fn assert_source_file(input: TokenStream) -> TokenStream { for tk in input { - let source_file = tk.span.source_file(); + let source_file = tk.span().source_file(); assert!(source_file.is_real(), "Source file is not real: {:?}", source_file); } diff --git a/src/test/ui-fulldeps/proc-macro/auxiliary/parent-source-spans.rs b/src/test/ui-fulldeps/proc-macro/auxiliary/parent-source-spans.rs index 3eb96c2ab9622..ed11b2db2f5ff 100644 --- a/src/test/ui-fulldeps/proc-macro/auxiliary/parent-source-spans.rs +++ b/src/test/ui-fulldeps/proc-macro/auxiliary/parent-source-spans.rs @@ -14,12 +14,12 @@ extern crate proc_macro; -use proc_macro::{TokenStream, TokenTree, TokenNode, Span}; +use proc_macro::{TokenStream, TokenTree, Span}; fn lit_span(tt: TokenTree) -> (Span, String) { - use TokenNode::*; - match tt.kind { - Literal(..) | Group(..) => (tt.span, tt.to_string().trim().into()), + match tt { + TokenTree::Literal(..) | + TokenTree::Group(..) => (tt.span(), tt.to_string().trim().into()), _ => panic!("expected a literal in token tree, got: {:?}", tt) } } diff --git a/src/test/ui-fulldeps/proc-macro/auxiliary/three-equals.rs b/src/test/ui-fulldeps/proc-macro/auxiliary/three-equals.rs index 2381c61b87bd0..fda0e28891f26 100644 --- a/src/test/ui-fulldeps/proc-macro/auxiliary/three-equals.rs +++ b/src/test/ui-fulldeps/proc-macro/auxiliary/three-equals.rs @@ -14,26 +14,27 @@ extern crate proc_macro; -use proc_macro::{TokenStream, TokenNode, Span, Diagnostic}; +use proc_macro::{TokenStream, TokenTree, Span, Diagnostic}; fn parse(input: TokenStream) -> Result<(), Diagnostic> { let mut count = 0; let mut last_span = Span::def_site(); for tree in input { - let span = tree.span; + let span = tree.span(); if count >= 3 { return Err(span.error(format!("expected EOF, found `{}`.", tree)) .span_note(last_span, "last good input was here") .help("input must be: `===`")) } - if let TokenNode::Op('=', _) = tree.kind { - count += 1; - } else { - return Err(span.error(format!("expected `=`, found `{}`.", tree))); + if let TokenTree::Op(tt) = tree { + if tt.op() == '=' { + count += 1; + last_span = span; + continue + } } - - last_span = span; + return Err(span.error(format!("expected `=`, found `{}`.", tree))); } if count < 3 { From c4c521457b724e147ab37fa17a42010dddc1c564 Mon Sep 17 00:00:00 2001 From: Wim Looman Date: Tue, 3 Apr 2018 14:29:28 +0200 Subject: [PATCH 06/12] impl Unpin for Pin --- src/libcore/mem.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/libcore/mem.rs b/src/libcore/mem.rs index b2467c948b4b1..e3f08926610f6 100644 --- a/src/libcore/mem.rs +++ b/src/libcore/mem.rs @@ -1213,3 +1213,6 @@ impl<'a, T: ?Sized> fmt::Pointer for Pin<'a, T> { #[unstable(feature = "pin", issue = "49150")] impl<'a, T: ?Sized + Unsize, U: ?Sized> CoerceUnsized> for Pin<'a, T> {} + +#[unstable(feature = "pin", issue = "49150")] +unsafe impl<'a, T: ?Sized> Unpin for Pin<'a, T> {} From 5d74990cebb82b9573ea6a9d509bb8e05fd6681e Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Sat, 10 Mar 2018 18:16:26 -0800 Subject: [PATCH 07/12] expand macro invocations in `extern {}` blocks --- src/librustc/hir/lowering.rs | 1 + src/librustc/hir/map/def_collector.rs | 4 + src/librustc_passes/ast_validation.rs | 18 ++- src/librustc_resolve/build_reduced_graph.rs | 6 + src/librustc_resolve/lib.rs | 1 + src/librustc_save_analysis/dump_visitor.rs | 1 + src/librustc_save_analysis/lib.rs | 1 + src/librustc_save_analysis/sig.rs | 1 + src/libsyntax/ast.rs | 3 + src/libsyntax/ext/base.rs | 28 ++++ src/libsyntax/ext/expand.rs | 58 ++++++++ src/libsyntax/ext/placeholders.rs | 11 ++ src/libsyntax/feature_gate.rs | 11 ++ src/libsyntax/fold.rs | 20 ++- src/libsyntax/parse/parser.rs | 124 ++++++++++-------- src/libsyntax/parse/token.rs | 2 + src/libsyntax/print/pprust.rs | 9 ++ src/libsyntax/visit.rs | 1 + src/libsyntax_ext/deriving/custom.rs | 1 + .../auxiliary/macro_crate_test.rs | 6 +- .../proc-macro/auxiliary/test-macros.rs | 36 +++++ .../proc-macro/macros-in-extern.rs | 38 ++++++ src/test/compile-fail/macros-in-extern.rs | 42 ++++++ src/test/parse-fail/duplicate-visibility.rs | 2 +- src/test/parse-fail/extern-no-fn.rs | 4 +- .../auxiliary/macro_crate_test.rs | 8 +- .../proc-macro/auxiliary/test-macros.rs | 36 +++++ .../proc-macro/macros-in-extern.rs | 35 +++++ src/test/run-pass/macros-in-extern.rs | 39 ++++++ src/test/ui/feature-gate-macros_in_extern.rs | 35 +++++ .../ui/feature-gate-macros_in_extern.stderr | 27 ++++ 31 files changed, 544 insertions(+), 65 deletions(-) create mode 100644 src/test/compile-fail-fulldeps/proc-macro/auxiliary/test-macros.rs create mode 100644 src/test/compile-fail-fulldeps/proc-macro/macros-in-extern.rs create mode 100644 src/test/compile-fail/macros-in-extern.rs create mode 100644 src/test/run-pass-fulldeps/proc-macro/auxiliary/test-macros.rs create mode 100644 src/test/run-pass-fulldeps/proc-macro/macros-in-extern.rs create mode 100644 src/test/run-pass/macros-in-extern.rs create mode 100644 src/test/ui/feature-gate-macros_in_extern.rs create mode 100644 src/test/ui/feature-gate-macros_in_extern.stderr diff --git a/src/librustc/hir/lowering.rs b/src/librustc/hir/lowering.rs index 536d682566a72..bbc1b9211886a 100644 --- a/src/librustc/hir/lowering.rs +++ b/src/librustc/hir/lowering.rs @@ -2724,6 +2724,7 @@ impl<'a> LoweringContext<'a> { hir::ForeignItemStatic(this.lower_ty(t, ImplTraitContext::Disallowed), m) } ForeignItemKind::Ty => hir::ForeignItemType, + ForeignItemKind::Macro(_) => panic!("shouldn't exist here"), }, vis: this.lower_visibility(&i.vis, None), span: i.span, diff --git a/src/librustc/hir/map/def_collector.rs b/src/librustc/hir/map/def_collector.rs index 3619a7fb0c603..20f46cb348d1d 100644 --- a/src/librustc/hir/map/def_collector.rs +++ b/src/librustc/hir/map/def_collector.rs @@ -181,6 +181,10 @@ impl<'a> visit::Visitor<'a> for DefCollector<'a> { } fn visit_foreign_item(&mut self, foreign_item: &'a ForeignItem) { + if let ForeignItemKind::Macro(_) = foreign_item.node { + return self.visit_macro_invoc(foreign_item.id, false); + } + let def = self.create_def(foreign_item.id, DefPathData::ValueNs(foreign_item.ident.name.as_str()), REGULAR_SPACE, diff --git a/src/librustc_passes/ast_validation.rs b/src/librustc_passes/ast_validation.rs index 37274d1fc4479..b34decc1c69a5 100644 --- a/src/librustc_passes/ast_validation.rs +++ b/src/librustc_passes/ast_validation.rs @@ -381,7 +381,7 @@ impl<'a> Visitor<'a> for AstValidator<'a> { .span_label(span, "pattern not allowed in foreign function").emit(); }); } - ForeignItemKind::Static(..) | ForeignItemKind::Ty => {} + ForeignItemKind::Static(..) | ForeignItemKind::Ty | ForeignItemKind::Macro(..) => {} } visit::walk_foreign_item(self, fi) @@ -460,6 +460,14 @@ impl<'a> Visitor<'a> for AstValidator<'a> { self.check_late_bound_lifetime_defs(&t.bound_generic_params); visit::walk_poly_trait_ref(self, t, m); } + + fn visit_mac(&mut self, mac: &Spanned) { + // when a new macro kind is added but the author forgets to set it up for expansion + // because that's the only part that won't cause a compiler error + self.session.diagnostic() + .span_bug(mac.span, "macro invocation missed in expansion; did you forget to override \ + the relevant `fold_*()` method in `PlaceholderExpander`?"); + } } // Bans nested `impl Trait`, e.g. `impl Into`. @@ -522,6 +530,10 @@ impl<'a> Visitor<'a> for NestedImplTraitVisitor<'a> { } } } + + fn visit_mac(&mut self, _mac: &Spanned) { + // covered in AstValidator + } } // Bans `impl Trait` in path projections like `::Item` or `Foo::Bar`. @@ -583,6 +595,10 @@ impl<'a> Visitor<'a> for ImplTraitProjectionVisitor<'a> { _ => visit::walk_ty(self, t), } } + + fn visit_mac(&mut self, _mac: &Spanned) { + // covered in AstValidator + } } pub fn check_crate(session: &Session, krate: &Crate) { diff --git a/src/librustc_resolve/build_reduced_graph.rs b/src/librustc_resolve/build_reduced_graph.rs index c192f349c2019..397590012fd73 100644 --- a/src/librustc_resolve/build_reduced_graph.rs +++ b/src/librustc_resolve/build_reduced_graph.rs @@ -456,6 +456,7 @@ impl<'a> Resolver<'a> { ForeignItemKind::Ty => { (Def::TyForeign(self.definitions.local_def_id(item.id)), TypeNS) } + ForeignItemKind::Macro(_) => unreachable!(), }; let parent = self.current_module; let vis = self.resolve_visibility(&item.vis); @@ -816,6 +817,11 @@ impl<'a, 'b> Visitor<'a> for BuildReducedGraphVisitor<'a, 'b> { } fn visit_foreign_item(&mut self, foreign_item: &'a ForeignItem) { + if let ForeignItemKind::Macro(_) = foreign_item.node { + self.visit_invoc(foreign_item.id); + return; + } + self.resolver.build_reduced_graph_for_foreign_item(foreign_item, self.expansion); visit::walk_foreign_item(self, foreign_item); } diff --git a/src/librustc_resolve/lib.rs b/src/librustc_resolve/lib.rs index 97dcf081f8c8d..74cfe57665274 100644 --- a/src/librustc_resolve/lib.rs +++ b/src/librustc_resolve/lib.rs @@ -863,6 +863,7 @@ impl<'a, 'tcx> Visitor<'tcx> for Resolver<'a> { } ForeignItemKind::Static(..) => NoTypeParameters, ForeignItemKind::Ty => NoTypeParameters, + ForeignItemKind::Macro(..) => NoTypeParameters, }; self.with_type_parameter_rib(type_parameters, |this| { visit::walk_foreign_item(this, foreign_item); diff --git a/src/librustc_save_analysis/dump_visitor.rs b/src/librustc_save_analysis/dump_visitor.rs index 3d4d8571c6e42..5e51797d72066 100644 --- a/src/librustc_save_analysis/dump_visitor.rs +++ b/src/librustc_save_analysis/dump_visitor.rs @@ -1812,6 +1812,7 @@ impl<'l, 'tcx: 'l, 'll, O: DumpOutput + 'll> Visitor<'l> for DumpVisitor<'l, 'tc self.dumper.dump_def(&access, var_data); } } + ast::ForeignItemKind::Macro(..) => {} } } } diff --git a/src/librustc_save_analysis/lib.rs b/src/librustc_save_analysis/lib.rs index 953747756517d..fb4cb2afe5a66 100644 --- a/src/librustc_save_analysis/lib.rs +++ b/src/librustc_save_analysis/lib.rs @@ -182,6 +182,7 @@ impl<'l, 'tcx: 'l> SaveContext<'l, 'tcx> { } // FIXME(plietar): needs a new DefKind in rls-data ast::ForeignItemKind::Ty => None, + ast::ForeignItemKind::Macro(..) => None, } } diff --git a/src/librustc_save_analysis/sig.rs b/src/librustc_save_analysis/sig.rs index 0c890ce19d081..fd4d3e7638642 100644 --- a/src/librustc_save_analysis/sig.rs +++ b/src/librustc_save_analysis/sig.rs @@ -822,6 +822,7 @@ impl Sig for ast::ForeignItem { refs: vec![], }) } + ast::ForeignItemKind::Macro(..) => Err("macro"), } } } diff --git a/src/libsyntax/ast.rs b/src/libsyntax/ast.rs index c90b0aecfc044..b55a502d11851 100644 --- a/src/libsyntax/ast.rs +++ b/src/libsyntax/ast.rs @@ -2195,6 +2195,8 @@ pub enum ForeignItemKind { Static(P, bool), /// A foreign type Ty, + /// A macro invocation + Macro(Mac), } impl ForeignItemKind { @@ -2203,6 +2205,7 @@ impl ForeignItemKind { ForeignItemKind::Fn(..) => "foreign function", ForeignItemKind::Static(..) => "foreign static item", ForeignItemKind::Ty => "foreign type", + ForeignItemKind::Macro(..) => "macro in foreign module", } } } diff --git a/src/libsyntax/ext/base.rs b/src/libsyntax/ext/base.rs index d3157af984e80..5a735be55c0e6 100644 --- a/src/libsyntax/ext/base.rs +++ b/src/libsyntax/ext/base.rs @@ -38,6 +38,7 @@ pub enum Annotatable { Item(P), TraitItem(P), ImplItem(P), + ForeignItem(P), Stmt(P), Expr(P), } @@ -48,6 +49,7 @@ impl HasAttrs for Annotatable { Annotatable::Item(ref item) => &item.attrs, Annotatable::TraitItem(ref trait_item) => &trait_item.attrs, Annotatable::ImplItem(ref impl_item) => &impl_item.attrs, + Annotatable::ForeignItem(ref foreign_item) => &foreign_item.attrs, Annotatable::Stmt(ref stmt) => stmt.attrs(), Annotatable::Expr(ref expr) => &expr.attrs, } @@ -58,6 +60,8 @@ impl HasAttrs for Annotatable { Annotatable::Item(item) => Annotatable::Item(item.map_attrs(f)), Annotatable::TraitItem(trait_item) => Annotatable::TraitItem(trait_item.map_attrs(f)), Annotatable::ImplItem(impl_item) => Annotatable::ImplItem(impl_item.map_attrs(f)), + Annotatable::ForeignItem(foreign_item) => + Annotatable::ForeignItem(foreign_item.map_attrs(f)), Annotatable::Stmt(stmt) => Annotatable::Stmt(stmt.map_attrs(f)), Annotatable::Expr(expr) => Annotatable::Expr(expr.map_attrs(f)), } @@ -70,6 +74,7 @@ impl Annotatable { Annotatable::Item(ref item) => item.span, Annotatable::TraitItem(ref trait_item) => trait_item.span, Annotatable::ImplItem(ref impl_item) => impl_item.span, + Annotatable::ForeignItem(ref foreign_item) => foreign_item.span, Annotatable::Stmt(ref stmt) => stmt.span, Annotatable::Expr(ref expr) => expr.span, } @@ -106,6 +111,13 @@ impl Annotatable { } } + pub fn expect_foreign_item(self) -> ast::ForeignItem { + match self { + Annotatable::ForeignItem(i) => i.into_inner(), + _ => panic!("expected foreign item") + } + } + pub fn derive_allowed(&self) -> bool { match *self { Annotatable::Item(ref item) => match item.node { @@ -317,6 +329,9 @@ pub trait MacResult { None } + /// Create zero or more items in an `extern {}` block + fn make_foreign_items(self: Box) -> Option> { None } + /// Create a pattern. fn make_pat(self: Box) -> Option> { None @@ -365,6 +380,7 @@ make_MacEager! { items: SmallVector>, impl_items: SmallVector, trait_items: SmallVector, + foreign_items: SmallVector, stmts: SmallVector, ty: P, } @@ -386,6 +402,10 @@ impl MacResult for MacEager { self.trait_items } + fn make_foreign_items(self: Box) -> Option> { + self.foreign_items + } + fn make_stmts(self: Box) -> Option> { match self.stmts.as_ref().map_or(0, |s| s.len()) { 0 => make_stmts_default!(self), @@ -502,6 +522,14 @@ impl MacResult for DummyResult { } } + fn make_foreign_items(self: Box) -> Option> { + if self.expr_only { + None + } else { + Some(SmallVector::new()) + } + } + fn make_stmts(self: Box) -> Option> { Some(SmallVector::one(ast::Stmt { id: ast::DUMMY_NODE_ID, diff --git a/src/libsyntax/ext/expand.rs b/src/libsyntax/ext/expand.rs index 864969c40750b..105de13b976a1 100644 --- a/src/libsyntax/ext/expand.rs +++ b/src/libsyntax/ext/expand.rs @@ -133,6 +133,8 @@ expansions! { "trait item", .make_trait_items, lift .fold_trait_item, lift .visit_trait_item; ImplItems: SmallVector [SmallVector, ast::ImplItem], "impl item", .make_impl_items, lift .fold_impl_item, lift .visit_impl_item; + ForeignItems: SmallVector [SmallVector, ast::ForeignItem], + "foreign item", .make_foreign_items, lift .fold_foreign_item, lift .visit_foreign_item; } impl ExpansionKind { @@ -149,6 +151,8 @@ impl ExpansionKind { Expansion::ImplItems(items.map(Annotatable::expect_impl_item).collect()), ExpansionKind::TraitItems => Expansion::TraitItems(items.map(Annotatable::expect_trait_item).collect()), + ExpansionKind::ForeignItems => + Expansion::ForeignItems(items.map(Annotatable::expect_foreign_item).collect()), _ => unreachable!(), } } @@ -435,6 +439,11 @@ impl<'a, 'b> MacroExpander<'a, 'b> { Annotatable::ImplItem(item) => { Annotatable::ImplItem(item.map(|item| cfg.fold_impl_item(item).pop().unwrap())) } + Annotatable::ForeignItem(item) => { + Annotatable::ForeignItem( + item.map(|item| cfg.fold_foreign_item(item).pop().unwrap()) + ) + } Annotatable::Stmt(stmt) => { Annotatable::Stmt(stmt.map(|stmt| cfg.fold_stmt(stmt).pop().unwrap())) } @@ -509,6 +518,7 @@ impl<'a, 'b> MacroExpander<'a, 'b> { Annotatable::Item(item) => token::NtItem(item), Annotatable::TraitItem(item) => token::NtTraitItem(item.into_inner()), Annotatable::ImplItem(item) => token::NtImplItem(item.into_inner()), + Annotatable::ForeignItem(item) => token::NtForeignItem(item.into_inner()), Annotatable::Stmt(stmt) => token::NtStmt(stmt.into_inner()), Annotatable::Expr(expr) => token::NtExpr(expr), })).into(); @@ -793,6 +803,15 @@ impl<'a> Parser<'a> { } Expansion::ImplItems(items) } + ExpansionKind::ForeignItems => { + let mut items = SmallVector::new(); + while self.token != token::Eof { + if let Some(item) = self.parse_foreign_item()? { + items.push(item); + } + } + Expansion::ForeignItems(items) + } ExpansionKind::Stmts => { let mut stmts = SmallVector::new(); while self.token != token::Eof && @@ -1166,6 +1185,44 @@ impl<'a, 'b> Folder for InvocationCollector<'a, 'b> { noop_fold_foreign_mod(self.cfg.configure_foreign_mod(foreign_mod), self) } + fn fold_foreign_item(&mut self, + foreign_item: ast::ForeignItem) -> SmallVector { + let (attr, traits, foreign_item) = self.classify_item(foreign_item); + + let explain = if self.cx.ecfg.proc_macro_enabled() { + feature_gate::EXPLAIN_PROC_MACROS_IN_EXTERN + } else { + feature_gate::EXPLAIN_MACROS_IN_EXTERN + }; + + if attr.is_some() || !traits.is_empty() { + if !self.cx.ecfg.macros_in_extern_enabled() { + if let Some(ref attr) = attr { + emit_feature_err(&self.cx.parse_sess, "macros_in_extern", attr.span, + GateIssue::Language, explain); + } + } + + let item = Annotatable::ForeignItem(P(foreign_item)); + return self.collect_attr(attr, traits, item, ExpansionKind::ForeignItems) + .make_foreign_items(); + } + + if let ast::ForeignItemKind::Macro(mac) = foreign_item.node { + self.check_attributes(&foreign_item.attrs); + + if !self.cx.ecfg.macros_in_extern_enabled() { + emit_feature_err(&self.cx.parse_sess, "macros_in_extern", foreign_item.span, + GateIssue::Language, explain); + } + + return self.collect_bang(mac, foreign_item.span, ExpansionKind::ForeignItems) + .make_foreign_items(); + } + + noop_fold_foreign_item(foreign_item, self) + } + fn fold_item_kind(&mut self, item: ast::ItemKind) -> ast::ItemKind { match item { ast::ItemKind::MacroDef(..) => item, @@ -1311,6 +1368,7 @@ impl<'feat> ExpansionConfig<'feat> { fn enable_allow_internal_unstable = allow_internal_unstable, fn enable_custom_derive = custom_derive, fn proc_macro_enabled = proc_macro, + fn macros_in_extern_enabled = macros_in_extern, } } diff --git a/src/libsyntax/ext/placeholders.rs b/src/libsyntax/ext/placeholders.rs index 9c2c22476e9d9..9f60882ca29fc 100644 --- a/src/libsyntax/ext/placeholders.rs +++ b/src/libsyntax/ext/placeholders.rs @@ -60,6 +60,10 @@ pub fn placeholder(kind: ExpansionKind, id: ast::NodeId) -> Expansion { defaultness: ast::Defaultness::Final, tokens: None, })), + ExpansionKind::ForeignItems => Expansion::ForeignItems(SmallVector::one(ast::ForeignItem { + id, span, ident, vis, attrs, + node: ast::ForeignItemKind::Macro(mac_placeholder()), + })), ExpansionKind::Pat => Expansion::Pat(P(ast::Pat { id, span, node: ast::PatKind::Mac(mac_placeholder()), })), @@ -132,6 +136,13 @@ impl<'a, 'b> Folder for PlaceholderExpander<'a, 'b> { } } + fn fold_foreign_item(&mut self, item: ast::ForeignItem) -> SmallVector { + match item.node { + ast::ForeignItemKind::Macro(_) => self.remove(item.id).make_foreign_items(), + _ => noop_fold_foreign_item(item, self), + } + } + fn fold_expr(&mut self, expr: P) -> P { match expr.node { ast::ExprKind::Mac(_) => self.remove(expr.id).make_expr(), diff --git a/src/libsyntax/feature_gate.rs b/src/libsyntax/feature_gate.rs index 463e76e14618a..d88305529ede4 100644 --- a/src/libsyntax/feature_gate.rs +++ b/src/libsyntax/feature_gate.rs @@ -447,6 +447,9 @@ declare_features! ( // Allows keywords to be escaped for use as identifiers (active, raw_identifiers, "1.26.0", Some(48589), None), + + // Allows macro invocations in `extern {}` blocks + (active, macros_in_extern, "1.27.0", Some(49476), None), ); declare_features! ( @@ -1296,6 +1299,13 @@ pub const EXPLAIN_UNSIZED_TUPLE_COERCION: &'static str = pub const EXPLAIN_MACRO_AT_MOST_ONCE_REP: &'static str = "Using the `?` macro Kleene operator for \"at most one\" repetition is unstable"; +pub const EXPLAIN_MACROS_IN_EXTERN: &'static str = + "Macro invocations in `extern {}` blocks are experimental."; + +// mention proc-macros when enabled +pub const EXPLAIN_PROC_MACROS_IN_EXTERN: &'static str = + "Macro and proc-macro invocations in `extern {}` blocks are experimental."; + struct PostExpansionVisitor<'a> { context: &'a Context<'a>, } @@ -1600,6 +1610,7 @@ impl<'a> Visitor<'a> for PostExpansionVisitor<'a> { gate_feature_post!(&self, extern_types, i.span, "extern types are experimental"); } + ast::ForeignItemKind::Macro(..) => {} } visit::walk_foreign_item(self, i) diff --git a/src/libsyntax/fold.rs b/src/libsyntax/fold.rs index 05a3150c139c9..72f695b4fdbc5 100644 --- a/src/libsyntax/fold.rs +++ b/src/libsyntax/fold.rs @@ -60,10 +60,14 @@ pub trait Folder : Sized { noop_fold_use_tree(use_tree, self) } - fn fold_foreign_item(&mut self, ni: ForeignItem) -> ForeignItem { + fn fold_foreign_item(&mut self, ni: ForeignItem) -> SmallVector { noop_fold_foreign_item(ni, self) } + fn fold_foreign_item_simple(&mut self, ni: ForeignItem) -> ForeignItem { + noop_fold_foreign_item_simple(ni, self) + } + fn fold_item(&mut self, i: P) -> SmallVector> { noop_fold_item(i, self) } @@ -414,7 +418,7 @@ pub fn noop_fold_foreign_mod(ForeignMod {abi, items}: ForeignMod, fld: &mut T) -> ForeignMod { ForeignMod { abi, - items: items.move_map(|x| fld.fold_foreign_item(x)), + items: items.move_flat_map(|x| fld.fold_foreign_item(x)), } } @@ -648,6 +652,10 @@ pub fn noop_fold_interpolated(nt: token::Nonterminal, fld: &mut T) token::NtArg(arg) => token::NtArg(fld.fold_arg(arg)), token::NtVis(vis) => token::NtVis(fld.fold_vis(vis)), token::NtLifetime(lifetime) => token::NtLifetime(fld.fold_lifetime(lifetime)), + token::NtForeignItem(ni) => + token::NtForeignItem(fld.fold_foreign_item(ni) + // see reasoning above + .expect_one("expected fold to produce exactly one item")), } } @@ -1072,7 +1080,12 @@ pub fn noop_fold_item_simple(Item {id, ident, attrs, node, vis, span, } } -pub fn noop_fold_foreign_item(ni: ForeignItem, folder: &mut T) -> ForeignItem { +pub fn noop_fold_foreign_item(ni: ForeignItem, folder: &mut T) +-> SmallVector { + SmallVector::one(folder.fold_foreign_item_simple(ni)) +} + +pub fn noop_fold_foreign_item_simple(ni: ForeignItem, folder: &mut T) -> ForeignItem { ForeignItem { id: folder.new_id(ni.id), vis: folder.fold_vis(ni.vis), @@ -1086,6 +1099,7 @@ pub fn noop_fold_foreign_item(ni: ForeignItem, folder: &mut T) -> For ForeignItemKind::Static(folder.fold_ty(t), m) } ForeignItemKind::Ty => ForeignItemKind::Ty, + ForeignItemKind::Macro(mac) => ForeignItemKind::Macro(folder.fold_mac(mac)), }, span: folder.new_span(ni.span) } diff --git a/src/libsyntax/parse/parser.rs b/src/libsyntax/parse/parser.rs index f7cdd4ba2b449..6e44fe7344f61 100644 --- a/src/libsyntax/parse/parser.rs +++ b/src/libsyntax/parse/parser.rs @@ -26,7 +26,7 @@ use ast::{Ident, ImplItem, IsAuto, Item, ItemKind}; use ast::{Label, Lifetime, LifetimeDef, Lit, LitKind, UintTy}; use ast::Local; use ast::MacStmtStyle; -use ast::Mac_; +use ast::{Mac, Mac_}; use ast::{MutTy, Mutability}; use ast::{Pat, PatKind, PathSegment}; use ast::{PolyTraitRef, QSelf}; @@ -1417,28 +1417,8 @@ impl<'a> Parser<'a> { None }; (ident, TraitItemKind::Const(ty, default), ast::Generics::default()) - } else if self.token.is_path_start() && !self.is_extern_non_path() { + } else if let Some(mac) = self.parse_assoc_macro_invoc("trait", None, &mut false)? { // trait item macro. - // code copied from parse_macro_use_or_failure... abstraction! - let prev_span = self.prev_span; - let lo = self.span; - let pth = self.parse_path(PathStyle::Mod)?; - - if pth.segments.len() == 1 { - if !self.eat(&token::Not) { - return Err(self.missing_assoc_item_kind_err("trait", prev_span)); - } - } else { - self.expect(&token::Not)?; - } - - // eat a matched-delimiter token tree: - let (delim, tts) = self.expect_delimited_token_tree()?; - if delim != token::Brace { - self.expect(&token::Semi)? - } - - let mac = respan(lo.to(self.prev_span), Mac_ { path: pth, tts: tts }); (keywords::Invalid.ident(), ast::TraitItemKind::Macro(mac), ast::Generics::default()) } else { let (constness, unsafety, abi) = self.parse_fn_front_matter()?; @@ -5406,6 +5386,12 @@ impl<'a> Parser<'a> { fn missing_assoc_item_kind_err(&mut self, item_type: &str, prev_span: Span) -> DiagnosticBuilder<'a> { + let expected_kinds = if item_type == "extern" { + "missing `fn`, `type`, or `static`" + } else { + "missing `fn`, `type`, or `const`" + }; + // Given this code `path(`, it seems like this is not // setting the visibility of a macro invocation, but rather // a mistyped method declaration. @@ -5418,9 +5404,9 @@ impl<'a> Parser<'a> { let sp = prev_span.between(self.prev_span); let mut err = self.diagnostic().struct_span_err( sp, - &format!("missing `fn`, `type`, or `const` for {}-item declaration", - item_type)); - err.span_label(sp, "missing `fn`, `type`, or `const`"); + &format!("{} for {}-item declaration", + expected_kinds, item_type)); + err.span_label(sp, expected_kinds); err } @@ -5429,31 +5415,8 @@ impl<'a> Parser<'a> { -> PResult<'a, (Ident, Vec, ast::Generics, ast::ImplItemKind)> { // code copied from parse_macro_use_or_failure... abstraction! - if self.token.is_path_start() && !self.is_extern_non_path() { + if let Some(mac) = self.parse_assoc_macro_invoc("impl", Some(vis), at_end)? { // Method macro. - - let prev_span = self.prev_span; - - let lo = self.span; - let pth = self.parse_path(PathStyle::Mod)?; - if pth.segments.len() == 1 { - if !self.eat(&token::Not) { - return Err(self.missing_assoc_item_kind_err("impl", prev_span)); - } - } else { - self.expect(&token::Not)?; - } - - self.complain_if_pub_macro(&vis.node, prev_span); - - // eat a matched-delimiter token tree: - *at_end = true; - let (delim, tts) = self.expect_delimited_token_tree()?; - if delim != token::Brace { - self.expect(&token::Semi)? - } - - let mac = respan(lo.to(self.prev_span), Mac_ { path: pth, tts: tts }); Ok((keywords::Invalid.ident(), vec![], ast::Generics::default(), ast::ImplItemKind::Macro(mac))) } else { @@ -6799,7 +6762,9 @@ impl<'a> Parser<'a> { } /// Parse a foreign item. - fn parse_foreign_item(&mut self) -> PResult<'a, Option> { + pub fn parse_foreign_item(&mut self) -> PResult<'a, Option> { + maybe_whole!(self, NtForeignItem, |ni| Some(ni)); + let attrs = self.parse_outer_attributes()?; let lo = self.span; let visibility = self.parse_visibility(false)?; @@ -6825,12 +6790,26 @@ impl<'a> Parser<'a> { return Ok(Some(self.parse_item_foreign_type(visibility, lo, attrs)?)); } - // FIXME #5668: this will occur for a macro invocation: - match self.parse_macro_use_or_failure(attrs, true, false, lo, visibility)? { - Some(item) => { - return Err(self.span_fatal(item.span, "macros cannot expand to foreign items")); + match self.parse_assoc_macro_invoc("extern", Some(&visibility), &mut false)? { + Some(mac) => { + Ok(Some( + ForeignItem { + ident: keywords::Invalid.ident(), + span: lo.to(self.prev_span), + id: ast::DUMMY_NODE_ID, + attrs, + vis: visibility, + node: ForeignItemKind::Macro(mac), + } + )) + } + None => { + if !attrs.is_empty() { + self.expected_item_err(&attrs); + } + + Ok(None) } - None => Ok(None) } } @@ -6894,6 +6873,41 @@ impl<'a> Parser<'a> { Ok(None) } + /// Parse a macro invocation inside a `trait`, `impl` or `extern` block + fn parse_assoc_macro_invoc(&mut self, item_kind: &str, vis: Option<&Visibility>, + at_end: &mut bool) -> PResult<'a, Option> + { + if self.token.is_path_start() && !self.is_extern_non_path() { + let prev_span = self.prev_span; + let lo = self.span; + let pth = self.parse_path(PathStyle::Mod)?; + + if pth.segments.len() == 1 { + if !self.eat(&token::Not) { + return Err(self.missing_assoc_item_kind_err(item_kind, prev_span)); + } + } else { + self.expect(&token::Not)?; + } + + if let Some(vis) = vis { + self.complain_if_pub_macro(&vis.node, prev_span); + } + + *at_end = true; + + // eat a matched-delimiter token tree: + let (delim, tts) = self.expect_delimited_token_tree()?; + if delim != token::Brace { + self.expect(&token::Semi)? + } + + Ok(Some(respan(lo.to(self.prev_span), Mac_ { path: pth, tts: tts }))) + } else { + Ok(None) + } + } + fn collect_tokens(&mut self, f: F) -> PResult<'a, (R, TokenStream)> where F: FnOnce(&mut Self) -> PResult<'a, R> { diff --git a/src/libsyntax/parse/token.rs b/src/libsyntax/parse/token.rs index e2dfca5d10a3c..5fe2b081566a1 100644 --- a/src/libsyntax/parse/token.rs +++ b/src/libsyntax/parse/token.rs @@ -581,6 +581,7 @@ pub enum Nonterminal { NtArm(ast::Arm), NtImplItem(ast::ImplItem), NtTraitItem(ast::TraitItem), + NtForeignItem(ast::ForeignItem), NtGenerics(ast::Generics), NtWhereClause(ast::WhereClause), NtArg(ast::Arg), @@ -603,6 +604,7 @@ impl fmt::Debug for Nonterminal { NtArm(..) => f.pad("NtArm(..)"), NtImplItem(..) => f.pad("NtImplItem(..)"), NtTraitItem(..) => f.pad("NtTraitItem(..)"), + NtForeignItem(..) => f.pad("NtForeignItem(..)"), NtGenerics(..) => f.pad("NtGenerics(..)"), NtWhereClause(..) => f.pad("NtWhereClause(..)"), NtArg(..) => f.pad("NtArg(..)"), diff --git a/src/libsyntax/print/pprust.rs b/src/libsyntax/print/pprust.rs index ae045fc095a50..fbcd251f1085e 100644 --- a/src/libsyntax/print/pprust.rs +++ b/src/libsyntax/print/pprust.rs @@ -281,6 +281,7 @@ pub fn token_to_string(tok: &Token) -> String { token::NtArg(ref e) => arg_to_string(e), token::NtVis(ref e) => vis_to_string(e), token::NtLifetime(ref e) => lifetime_to_string(e), + token::NtForeignItem(ref ni) => foreign_item_to_string(ni), } } } @@ -422,6 +423,10 @@ pub fn mac_to_string(arg: &ast::Mac) -> String { to_string(|s| s.print_mac(arg, ::parse::token::Paren)) } +pub fn foreign_item_to_string(arg: &ast::ForeignItem) -> String { + to_string(|s| s.print_foreign_item(arg)) +} + pub fn visibility_qualified(vis: &ast::Visibility, s: &str) -> String { format!("{}{}", to_string(|s| s.print_visibility(vis)), s) } @@ -1127,6 +1132,10 @@ impl<'a> State<'a> { self.end()?; // end the head-ibox self.end() // end the outer cbox } + ast::ForeignItemKind::Macro(ref m) => { + self.print_mac(m, token::Paren)?; + self.s.word(";") + } } } diff --git a/src/libsyntax/visit.rs b/src/libsyntax/visit.rs index bbf1fe124f1ba..9004ad022f701 100644 --- a/src/libsyntax/visit.rs +++ b/src/libsyntax/visit.rs @@ -460,6 +460,7 @@ pub fn walk_foreign_item<'a, V: Visitor<'a>>(visitor: &mut V, foreign_item: &'a } ForeignItemKind::Static(ref typ, _) => visitor.visit_ty(typ), ForeignItemKind::Ty => (), + ForeignItemKind::Macro(ref mac) => visitor.visit_mac(mac), } walk_list!(visitor, visit_attribute, &foreign_item.attrs); diff --git a/src/libsyntax_ext/deriving/custom.rs b/src/libsyntax_ext/deriving/custom.rs index 80557078d5463..5fd5e29948852 100644 --- a/src/libsyntax_ext/deriving/custom.rs +++ b/src/libsyntax_ext/deriving/custom.rs @@ -55,6 +55,7 @@ impl MultiItemModifier for ProcMacroDerive { Annotatable::Item(item) => item, Annotatable::ImplItem(_) | Annotatable::TraitItem(_) | + Annotatable::ForeignItem(_) | Annotatable::Stmt(_) | Annotatable::Expr(_) => { ecx.span_err(span, "proc-macro derives may only be \ diff --git a/src/test/compile-fail-fulldeps/auxiliary/macro_crate_test.rs b/src/test/compile-fail-fulldeps/auxiliary/macro_crate_test.rs index 77ea301941938..bc51b4061ed6b 100644 --- a/src/test/compile-fail-fulldeps/auxiliary/macro_crate_test.rs +++ b/src/test/compile-fail-fulldeps/auxiliary/macro_crate_test.rs @@ -93,7 +93,9 @@ fn expand_into_foo_multi(cx: &mut ExtCtxt, } }) } - // these are covered in proc_macro/attr-stmt-expr.rs + // covered in proc_macro/macros-in-extern.rs + Annotatable::ForeignItem(_) => unimplemented!(), + // covered in proc_macro/attr-stmt-expr.rs Annotatable::Stmt(_) | Annotatable::Expr(_) => panic!("expected item") } } @@ -147,6 +149,8 @@ fn expand_duplicate(cx: &mut ExtCtxt, new_it.ident = copy_name; push(Annotatable::TraitItem(P(new_it))); } + // covered in proc_macro/macros-in-extern.rs + Annotatable::ForeignItem(_) => unimplemented!(), // covered in proc_macro/attr-stmt-expr.rs Annotatable::Stmt(_) | Annotatable::Expr(_) => panic!("expected item") } diff --git a/src/test/compile-fail-fulldeps/proc-macro/auxiliary/test-macros.rs b/src/test/compile-fail-fulldeps/proc-macro/auxiliary/test-macros.rs new file mode 100644 index 0000000000000..d1c5b9050aa87 --- /dev/null +++ b/src/test/compile-fail-fulldeps/proc-macro/auxiliary/test-macros.rs @@ -0,0 +1,36 @@ +// Copyright 2018 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +// no-prefer-dynamic + +#![crate_type = "proc-macro"] +#![feature(proc_macro)] + +extern crate proc_macro; + +use proc_macro::TokenStream; + +#[proc_macro_attribute] +pub fn nop_attr(_attr: TokenStream, input: TokenStream) -> TokenStream { + assert!(_attr.to_string().is_empty()); + input +} + +#[proc_macro_attribute] +pub fn no_output(_attr: TokenStream, _input: TokenStream) -> TokenStream { + assert!(_attr.to_string().is_empty()); + assert!(!_input.to_string().is_empty()); + "".parse().unwrap() +} + +#[proc_macro] +pub fn emit_input(input: TokenStream) -> TokenStream { + input +} diff --git a/src/test/compile-fail-fulldeps/proc-macro/macros-in-extern.rs b/src/test/compile-fail-fulldeps/proc-macro/macros-in-extern.rs new file mode 100644 index 0000000000000..4c88df332460b --- /dev/null +++ b/src/test/compile-fail-fulldeps/proc-macro/macros-in-extern.rs @@ -0,0 +1,38 @@ +// Copyright 2018 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +// aux-build:test-macros.rs +// ignore-stage1 +// ignore-wasm32 + +#![feature(proc_macro)] + +extern crate test_macros; + +use test_macros::{nop_attr, no_output, emit_input}; + +fn main() { + assert_eq!(unsafe { rust_get_test_int() }, 0isize); + assert_eq!(unsafe { rust_dbg_extern_identity_u32(0xDEADBEEF) }, 0xDEADBEEF); +} + +#[link(name = "rust_test_helpers", kind = "static")] +extern { + #[no_output] + //~^ ERROR Macro and proc-macro invocations in `extern {}` blocks are experimental. + fn some_definitely_unknown_symbol_which_should_be_removed(); + + #[nop_attr] + //~^ ERROR Macro and proc-macro invocations in `extern {}` blocks are experimental. + fn rust_get_test_int() -> isize; + + emit_input!(fn rust_dbg_extern_identity_u32(arg: u32) -> u32;); + //~^ ERROR Macro and proc-macro invocations in `extern {}` blocks are experimental. +} diff --git a/src/test/compile-fail/macros-in-extern.rs b/src/test/compile-fail/macros-in-extern.rs new file mode 100644 index 0000000000000..7d7f95cbbf5cb --- /dev/null +++ b/src/test/compile-fail/macros-in-extern.rs @@ -0,0 +1,42 @@ +// Copyright 2018 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +// ignore-wasm32 + +#![feature(decl_macro)] + +macro_rules! returns_isize( + ($ident:ident) => ( + fn $ident() -> isize; + ) +); + +macro takes_u32_returns_u32($ident:ident) { + fn $ident (arg: u32) -> u32; +} + +macro_rules! emits_nothing( + () => () +); + +fn main() { + assert_eq!(unsafe { rust_get_test_int() }, 0isize); + assert_eq!(unsafe { rust_dbg_extern_identity_u32(0xDEADBEEF) }, 0xDEADBEEFu32); +} + +#[link(name = "rust_test_helpers", kind = "static")] +extern { + returns_isize!(rust_get_test_int); + //~^ ERROR Macro invocations in `extern {}` blocks are experimental. + takes_u32_returns_u32!(rust_dbg_extern_identity_u32); + //~^ ERROR Macro invocations in `extern {}` blocks are experimental. + emits_nothing!(); + //~^ ERROR Macro invocations in `extern {}` blocks are experimental. +} diff --git a/src/test/parse-fail/duplicate-visibility.rs b/src/test/parse-fail/duplicate-visibility.rs index 5ee84cd5543ba..6899caa7153bb 100644 --- a/src/test/parse-fail/duplicate-visibility.rs +++ b/src/test/parse-fail/duplicate-visibility.rs @@ -10,7 +10,7 @@ // compile-flags: -Z parse-only -// error-pattern:unmatched visibility `pub` +// error-pattern:expected one of `(`, `fn`, `static`, `type`, or `}` here extern { pub pub fn foo(); } diff --git a/src/test/parse-fail/extern-no-fn.rs b/src/test/parse-fail/extern-no-fn.rs index ff3fefde40ece..aa0dbd4d4fc1b 100644 --- a/src/test/parse-fail/extern-no-fn.rs +++ b/src/test/parse-fail/extern-no-fn.rs @@ -10,8 +10,8 @@ // compile-flags: -Z parse-only -extern { - f(); //~ ERROR expected one of `!` or `::`, found `(` +extern { //~ ERROR missing `fn`, `type`, or `static` for extern-item declaration + f(); } fn main() { diff --git a/src/test/run-pass-fulldeps/auxiliary/macro_crate_test.rs b/src/test/run-pass-fulldeps/auxiliary/macro_crate_test.rs index 5ebd32921329d..6612fe45b81d3 100644 --- a/src/test/run-pass-fulldeps/auxiliary/macro_crate_test.rs +++ b/src/test/run-pass-fulldeps/auxiliary/macro_crate_test.rs @@ -96,7 +96,9 @@ fn expand_into_foo_multi(cx: &mut ExtCtxt, } }) ], - // these are covered in proc_macro/attr-stmt-expr.rs + // covered in proc_macro/macros-in-extern.rs + Annotatable::ForeignItem(..) => unimplemented!(), + // covered in proc_macro/attr-stmt-expr.rs Annotatable::Stmt(_) | Annotatable::Expr(_) => panic!("expected item"), } } @@ -142,7 +144,9 @@ fn expand_duplicate(cx: &mut ExtCtxt, new_it.ident = copy_name; push(Annotatable::TraitItem(P(new_it))); } - // these are covered in proc_macro/attr-stmt-expr.rs + // covered in proc_macro/macros-in-extern.rs + Annotatable::ForeignItem(..) => unimplemented!(), + // covered in proc_macro/attr-stmt-expr.rs Annotatable::Stmt(_) | Annotatable::Expr(_) => panic!("expected item") } } diff --git a/src/test/run-pass-fulldeps/proc-macro/auxiliary/test-macros.rs b/src/test/run-pass-fulldeps/proc-macro/auxiliary/test-macros.rs new file mode 100644 index 0000000000000..d1c5b9050aa87 --- /dev/null +++ b/src/test/run-pass-fulldeps/proc-macro/auxiliary/test-macros.rs @@ -0,0 +1,36 @@ +// Copyright 2018 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +// no-prefer-dynamic + +#![crate_type = "proc-macro"] +#![feature(proc_macro)] + +extern crate proc_macro; + +use proc_macro::TokenStream; + +#[proc_macro_attribute] +pub fn nop_attr(_attr: TokenStream, input: TokenStream) -> TokenStream { + assert!(_attr.to_string().is_empty()); + input +} + +#[proc_macro_attribute] +pub fn no_output(_attr: TokenStream, _input: TokenStream) -> TokenStream { + assert!(_attr.to_string().is_empty()); + assert!(!_input.to_string().is_empty()); + "".parse().unwrap() +} + +#[proc_macro] +pub fn emit_input(input: TokenStream) -> TokenStream { + input +} diff --git a/src/test/run-pass-fulldeps/proc-macro/macros-in-extern.rs b/src/test/run-pass-fulldeps/proc-macro/macros-in-extern.rs new file mode 100644 index 0000000000000..59b9b0baa8a45 --- /dev/null +++ b/src/test/run-pass-fulldeps/proc-macro/macros-in-extern.rs @@ -0,0 +1,35 @@ +// Copyright 2018 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +// aux-build:test-macros.rs +// ignore-stage1 +// ignore-wasm32 + +#![feature(proc_macro, macros_in_extern)] + +extern crate test_macros; + +use test_macros::{nop_attr, no_output, emit_input}; + +fn main() { + assert_eq!(unsafe { rust_get_test_int() }, 1isize); + assert_eq!(unsafe { rust_dbg_extern_identity_u32(0xDEADBEEF) }, 0xDEADBEEF); +} + +#[link(name = "rust_test_helpers", kind = "static")] +extern { + #[no_output] + fn some_definitely_unknown_symbol_which_should_be_removed(); + + #[nop_attr] + fn rust_get_test_int() -> isize; + + emit_input!(fn rust_dbg_extern_identity_u32(arg: u32) -> u32;); +} diff --git a/src/test/run-pass/macros-in-extern.rs b/src/test/run-pass/macros-in-extern.rs new file mode 100644 index 0000000000000..d9094934356fc --- /dev/null +++ b/src/test/run-pass/macros-in-extern.rs @@ -0,0 +1,39 @@ +// Copyright 2018 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +// ignore-wasm32 + +#![feature(decl_macro, macros_in_extern)] + +macro_rules! returns_isize( + ($ident:ident) => ( + fn $ident() -> isize; + ) +); + +macro takes_u32_returns_u32($ident:ident) { + fn $ident (arg: u32) -> u32; +} + +macro_rules! emits_nothing( + () => () +); + +fn main() { + assert_eq!(unsafe { rust_get_test_int() }, 1isize); + assert_eq!(unsafe { rust_dbg_extern_identity_u32(0xDEADBEEF) }, 0xDEADBEEFu32); +} + +#[link(name = "rust_test_helpers", kind = "static")] +extern { + returns_isize!(rust_get_test_int); + takes_u32_returns_u32!(rust_dbg_extern_identity_u32); + emits_nothing!(); +} diff --git a/src/test/ui/feature-gate-macros_in_extern.rs b/src/test/ui/feature-gate-macros_in_extern.rs new file mode 100644 index 0000000000000..9c758241ea1b8 --- /dev/null +++ b/src/test/ui/feature-gate-macros_in_extern.rs @@ -0,0 +1,35 @@ +// Copyright 2018 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +#![feature(decl_macro)] + +macro_rules! returns_isize( + ($ident:ident) => ( + fn $ident() -> isize; + ) +); + +macro takes_u32_returns_u32($ident:ident) { + fn $ident (arg: u32) -> u32; +} + +macro_rules! emits_nothing( + () => () +); + +#[link(name = "rust_test_helpers", kind = "static")] +extern { + returns_isize!(rust_get_test_int); + //~^ ERROR Macro invocations in `extern {}` blocks are experimental. + takes_u32_returns_u32!(rust_dbg_extern_identity_u32); + //~^ ERROR Macro invocations in `extern {}` blocks are experimental. + emits_nothing!(); + //~^ ERROR Macro invocations in `extern {}` blocks are experimental. +} diff --git a/src/test/ui/feature-gate-macros_in_extern.stderr b/src/test/ui/feature-gate-macros_in_extern.stderr new file mode 100644 index 0000000000000..49aca0db2d46c --- /dev/null +++ b/src/test/ui/feature-gate-macros_in_extern.stderr @@ -0,0 +1,27 @@ +error[E0658]: Macro invocations in `extern {}` blocks are experimental. (see issue #49476) + --> $DIR/feature-gate-macros_in_extern.rs:29:5 + | +LL | returns_isize!(rust_get_test_int); + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + | + = help: add #![feature(macros_in_extern)] to the crate attributes to enable + +error[E0658]: Macro invocations in `extern {}` blocks are experimental. (see issue #49476) + --> $DIR/feature-gate-macros_in_extern.rs:31:5 + | +LL | takes_u32_returns_u32!(rust_dbg_extern_identity_u32); + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + | + = help: add #![feature(macros_in_extern)] to the crate attributes to enable + +error[E0658]: Macro invocations in `extern {}` blocks are experimental. (see issue #49476) + --> $DIR/feature-gate-macros_in_extern.rs:33:5 + | +LL | emits_nothing!(); + | ^^^^^^^^^^^^^^^^^ + | + = help: add #![feature(macros_in_extern)] to the crate attributes to enable + +error: aborting due to 3 previous errors + +For more information about this error, try `rustc --explain E0658`. From a57b1fbd84543880e4cba99013c81b7b82292c31 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Wed, 4 Apr 2018 14:27:53 -0700 Subject: [PATCH 08/12] Tweak doc comment expansion * Expand `!` tokens for inner doc comments * Trim leading doc comment decoration in the string literal Both of these should help bring the expansion inline with what `macro_rules!` already does. Closes #49655 Closes #49656 --- src/libproc_macro/lib.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/libproc_macro/lib.rs b/src/libproc_macro/lib.rs index 837900f05d2e7..ebd5c834fd036 100644 --- a/src/libproc_macro/lib.rs +++ b/src/libproc_macro/lib.rs @@ -59,6 +59,7 @@ use syntax::errors::DiagnosticBuilder; use syntax::parse::{self, token}; use syntax::symbol::Symbol; use syntax::tokenstream; +use syntax::parse::lexer::comments; use syntax_pos::{FileMap, Pos, SyntaxContext, FileName}; use syntax_pos::hygiene::Mark; @@ -1056,12 +1057,17 @@ impl TokenTree { } Literal(..) => tt!(self::Literal { token, span: Span(span) }), DocComment(c) => { + let style = comments::doc_comment_style(&c.as_str()); + let stripped = comments::strip_doc_comment_decoration(&c.as_str()); let stream = vec![ tt!(Term::new("doc", Span(span))), tt!(Op::new('=', Spacing::Alone)), - tt!(self::Literal::string(&c.as_str())), + tt!(self::Literal::string(&stripped)), ].into_iter().collect(); stack.push(tt!(Group::new(Delimiter::Bracket, stream))); + if style == ast::AttrStyle::Inner { + stack.push(tt!(Op::new('!', Spacing::Alone))); + } tt!(Op::new('#', Spacing::Alone)) } From 64ddb390efb2143f11c1583d52c78da5a290e097 Mon Sep 17 00:00:00 2001 From: memoryleak47 Date: Thu, 5 Apr 2018 13:04:00 +0200 Subject: [PATCH 09/12] typos --- src/libstd/lib.rs | 2 +- src/libstd/panic.rs | 2 +- src/libstd/sys/windows/pipe.rs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libstd/lib.rs b/src/libstd/lib.rs index 36eb729182210..67ef47569d631 100644 --- a/src/libstd/lib.rs +++ b/src/libstd/lib.rs @@ -380,7 +380,7 @@ extern crate compiler_builtins; // During testing, this crate is not actually the "real" std library, but rather // it links to the real std library, which was compiled from this same source // code. So any lang items std defines are conditionally excluded (or else they -// wolud generate duplicate lang item errors), and any globals it defines are +// would generate duplicate lang item errors), and any globals it defines are // _not_ the globals used by "real" std. So this import, defined only during // testing gives test-std access to real-std lang items and globals. See #2912 #[cfg(test)] extern crate std as realstd; diff --git a/src/libstd/panic.rs b/src/libstd/panic.rs index 79857104b9b68..28c178307a510 100644 --- a/src/libstd/panic.rs +++ b/src/libstd/panic.rs @@ -188,7 +188,7 @@ pub struct AssertUnwindSafe( // * By default everything is unwind safe // * pointers T contains mutability of some form are not unwind safe // * Unique, an owning pointer, lifts an implementation -// * Types like Mutex/RwLock which are explicilty poisoned are unwind safe +// * Types like Mutex/RwLock which are explicitly poisoned are unwind safe // * Our custom AssertUnwindSafe wrapper is indeed unwind safe #[stable(feature = "catch_unwind", since = "1.9.0")] diff --git a/src/libstd/sys/windows/pipe.rs b/src/libstd/sys/windows/pipe.rs index f3b1185c6ea91..df1dd7401af61 100644 --- a/src/libstd/sys/windows/pipe.rs +++ b/src/libstd/sys/windows/pipe.rs @@ -236,7 +236,7 @@ enum State { impl<'a> AsyncPipe<'a> { fn new(pipe: Handle, dst: &'a mut Vec) -> io::Result> { // Create an event which we'll use to coordinate our overlapped - // opreations, this event will be used in WaitForMultipleObjects + // operations, this event will be used in WaitForMultipleObjects // and passed as part of the OVERLAPPED handle. // // Note that we do a somewhat clever thing here by flagging the From 649f431acfaff1bcaad56f63b19c28499fb75964 Mon Sep 17 00:00:00 2001 From: kennytm Date: Fri, 6 Apr 2018 01:09:59 +0800 Subject: [PATCH 10/12] Give a name to every CI job. Bots that read the log can simply look for `[CI_JOB_NAME=...]` to find out the job's name. --- .travis.yml | 7 ++++++- appveyor.yml | 14 ++++++++++++++ src/ci/docker/run.sh | 1 + src/ci/run.sh | 4 ++++ 4 files changed, 25 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 36329ab91143c..f36ad67b11103 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,7 +21,7 @@ matrix: # "alternate" deployments, these are "nightlies" but have LLVM assertions # turned on, they're deployed to a different location primarily for # additional testing. - - env: IMAGE=dist-x86_64-linux DEPLOY_ALT=1 + - env: IMAGE=dist-x86_64-linux DEPLOY_ALT=1 CI_JOB_NAME=dist-x86_64-linux-alt if: branch = try OR branch = auto - env: > @@ -33,6 +33,7 @@ matrix: MACOSX_DEPLOYMENT_TARGET=10.7 NO_LLVM_ASSERTIONS=1 NO_DEBUG_ASSERTIONS=1 + CI_JOB_NAME=dist-x86_64-apple-alt os: osx osx_image: xcode9.3-moar if: branch = auto @@ -53,6 +54,7 @@ matrix: MACOSX_STD_DEPLOYMENT_TARGET=10.7 NO_LLVM_ASSERTIONS=1 NO_DEBUG_ASSERTIONS=1 + CI_JOB_NAME=x86_64-apple os: osx osx_image: xcode9.3-moar if: branch = auto @@ -66,6 +68,7 @@ matrix: MACOSX_STD_DEPLOYMENT_TARGET=10.7 NO_LLVM_ASSERTIONS=1 NO_DEBUG_ASSERTIONS=1 + CI_JOB_NAME=i686-apple os: osx osx_image: xcode9.3-moar if: branch = auto @@ -85,6 +88,7 @@ matrix: MACOSX_DEPLOYMENT_TARGET=10.7 NO_LLVM_ASSERTIONS=1 NO_DEBUG_ASSERTIONS=1 + CI_JOB_NAME=dist-i686-apple os: osx osx_image: xcode9.3-moar if: branch = auto @@ -98,6 +102,7 @@ matrix: MACOSX_DEPLOYMENT_TARGET=10.7 NO_LLVM_ASSERTIONS=1 NO_DEBUG_ASSERTIONS=1 + CI_JOB_NAME=dist-x86_64-apple os: osx osx_image: xcode9.3-moar if: branch = auto diff --git a/appveyor.yml b/appveyor.yml index 09c6fca5d02af..a15f3dd8d5cac 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -18,25 +18,31 @@ environment: - MSYS_BITS: 64 RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler SCRIPT: python x.py test + CI_JOB_NAME: x86_64-msvc - MSYS_BITS: 32 RUST_CONFIGURE_ARGS: --build=i686-pc-windows-msvc SCRIPT: make appveyor-subset-1 + CI_JOB_NAME: i686-msvc-1 - MSYS_BITS: 32 RUST_CONFIGURE_ARGS: --build=i686-pc-windows-msvc SCRIPT: make appveyor-subset-2 + CI_JOB_NAME: i686-msvc-2 # MSVC aux tests - MSYS_BITS: 64 RUST_CHECK_TARGET: check-aux EXCLUDE_CARGO=1 RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc + CI_JOB_NAME: x86_64-msvc-aux - MSYS_BITS: 64 SCRIPT: python x.py test src/tools/cargotest src/tools/cargo RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc + CI_JOB_NAME: x86_64-msvc-cargo # MSVC tools tests - MSYS_BITS: 64 SCRIPT: src/ci/docker/x86_64-gnu-tools/checktools.sh x.py /tmp/toolstates.json windows RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --save-toolstates=/tmp/toolstates.json --enable-test-miri + CI_JOB_NAME: x86_64-msvc-tools # 32/64-bit MinGW builds. # @@ -57,18 +63,21 @@ environment: MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror MINGW_ARCHIVE: i686-6.3.0-release-posix-dwarf-rt_v5-rev2.7z MINGW_DIR: mingw32 + CI_JOB_NAME: i686-mingw-1 - MSYS_BITS: 32 RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu SCRIPT: make appveyor-subset-2 MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror MINGW_ARCHIVE: i686-6.3.0-release-posix-dwarf-rt_v5-rev2.7z MINGW_DIR: mingw32 + CI_JOB_NAME: i686-mingw-2 - MSYS_BITS: 64 SCRIPT: python x.py test RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror MINGW_ARCHIVE: x86_64-6.3.0-release-posix-seh-rt_v5-rev2.7z MINGW_DIR: mingw64 + CI_JOB_NAME: x86_64-mingw # 32/64 bit MSVC and GNU deployment - RUST_CONFIGURE_ARGS: > @@ -77,6 +86,7 @@ environment: --enable-profiler SCRIPT: python x.py dist DEPLOY: 1 + CI_JOB_NAME: dist-x86_64-msvc - RUST_CONFIGURE_ARGS: > --build=i686-pc-windows-msvc --target=i586-pc-windows-msvc @@ -84,6 +94,7 @@ environment: --enable-profiler SCRIPT: python x.py dist DEPLOY: 1 + CI_JOB_NAME: dist-i686-msvc - MSYS_BITS: 32 RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu --enable-full-tools SCRIPT: python x.py dist @@ -91,6 +102,7 @@ environment: MINGW_ARCHIVE: i686-6.3.0-release-posix-dwarf-rt_v5-rev2.7z MINGW_DIR: mingw32 DEPLOY: 1 + CI_JOB_NAME: dist-i686-mingw - MSYS_BITS: 64 SCRIPT: python x.py dist RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu --enable-full-tools @@ -98,12 +110,14 @@ environment: MINGW_ARCHIVE: x86_64-6.3.0-release-posix-seh-rt_v5-rev2.7z MINGW_DIR: mingw64 DEPLOY: 1 + CI_JOB_NAME: dist-x86_64-mingw # "alternate" deployment, see .travis.yml for more info - MSYS_BITS: 64 RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-extended --enable-profiler SCRIPT: python x.py dist DEPLOY_ALT: 1 + CI_JOB_NAME: dist-x86_64-msvc-alt matrix: fast_finish: true diff --git a/src/ci/docker/run.sh b/src/ci/docker/run.sh index ea2387b67dbcc..c470ae7eb3030 100755 --- a/src/ci/docker/run.sh +++ b/src/ci/docker/run.sh @@ -130,6 +130,7 @@ exec docker \ --env TRAVIS \ --env TRAVIS_BRANCH \ --env TOOLSTATE_REPO_ACCESS_TOKEN \ + --env CI_JOB_NAME="${CI_JOB_NAME-$IMAGE}" \ --volume "$HOME/.cargo:/cargo" \ --volume "$HOME/rustsrc:$HOME/rustsrc" \ --init \ diff --git a/src/ci/run.sh b/src/ci/run.sh index 44eae0d180047..119b239d6b290 100755 --- a/src/ci/run.sh +++ b/src/ci/run.sh @@ -11,6 +11,10 @@ set -e +if [ -n "$CI_JOB_NAME" ]; then + echo "[CI_JOB_NAME=$CI_JOB_NAME]" +fi + if [ "$NO_CHANGE_USER" = "" ]; then if [ "$LOCAL_USER_ID" != "" ]; then useradd --shell /bin/bash -u $LOCAL_USER_ID -o -c "" -m user From a29d4d9ad6fb27003712932566724be265e354cd Mon Sep 17 00:00:00 2001 From: Wim Looman Date: Thu, 5 Apr 2018 20:03:02 +0200 Subject: [PATCH 11/12] impl Unpin for PinBox --- src/liballoc/boxed.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/liballoc/boxed.rs b/src/liballoc/boxed.rs index e59a6e9fdea60..a319fb797f88b 100644 --- a/src/liballoc/boxed.rs +++ b/src/liballoc/boxed.rs @@ -994,3 +994,6 @@ impl fmt::Pointer for PinBox { #[unstable(feature = "pin", issue = "49150")] impl, U: ?Sized> CoerceUnsized> for PinBox {} + +#[unstable(feature = "pin", issue = "49150")] +unsafe impl Unpin for PinBox {} From 59059f2ed1d9ee6d23a8aba0507bac7ac6d40e48 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Thu, 5 Apr 2018 11:36:01 -0700 Subject: [PATCH 12/12] Filter out missing components from manifests This commit updates our manifest generation for rustup to filter out any components/extensions which are actually missing. This is intended to help mitigate #49462 by making the manifests reflect reality, that many targets now are missing a `rust-docs` component rather than requiring it exists. --- src/tools/build-manifest/src/main.rs | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/src/tools/build-manifest/src/main.rs b/src/tools/build-manifest/src/main.rs index 49b054f8b9e50..0f482c95e05a2 100644 --- a/src/tools/build-manifest/src/main.rs +++ b/src/tools/build-manifest/src/main.rs @@ -356,6 +356,28 @@ impl Builder { target: "*".to_string(), }); + // If the components/extensions don't actually exist for this + // particular host/target combination then nix it entirely from our + // lists. + { + let has_component = |c: &Component| { + if c.target == "*" { + return true + } + let pkg = match manifest.pkg.get(&c.pkg) { + Some(p) => p, + None => return false, + }; + let target = match pkg.target.get(&c.target) { + Some(t) => t, + None => return false, + }; + target.available + }; + extensions.retain(&has_component); + components.retain(&has_component); + } + pkg.target.insert(host.to_string(), Target { available: true, url: Some(self.url(&filename)),