Auto merge of #44529 - alexcrichton:trans-query, r=michaelwoerister

Refactor translation unit partitioning/collection as a query

This commit is targeted at #44486 with the ultimate goal of making the `collect_and_partition_translation_items` function a query. This mostly just involved query-ifying a few other systems along with plumbing the tcx instead of `SharedCrateContext` in a few locations.

Currently this only tackles the first bullet of #44486 and doesn't add a dedicated query for a particular codegen unit. I wasn't quite sure how to do that yet but figured this was good to put up.

Closes #44486

Author: bors

src/librustc/dep_graph/dep_node.rs

@@ -71,6 +71,7 @@ use rustc_data_structures::stable_hasher::{StableHasher, HashStable};
 use ich::StableHashingContext;
 use std::fmt;
 use std::hash::Hash;
+use syntax_pos::symbol::InternedString;
 
 // erase!() just makes tokens go away. It's used to specify which macro argument
 // is repeated (i.e. which sub-expression of the macro we are in) but don't need
@@ -535,7 +536,7 @@ define_dep_nodes!( <'tcx>
     [] GetPanicStrategy(CrateNum),
     [] IsNoBuiltins(CrateNum),
     [] ImplDefaultness(DefId),
-    [] ExportedSymbols(CrateNum),
+    [] ExportedSymbolIds(CrateNum),
     [] NativeLibraries(CrateNum),
     [] PluginRegistrarFn(CrateNum),
     [] DeriveRegistrarFn(CrateNum),
@@ -575,6 +576,14 @@ define_dep_nodes!( <'tcx>
     [] MaybeUnusedExternCrates,
     [] StabilityIndex,
     [] AllCrateNums,
+    [] ExportedSymbols(CrateNum),
+    [] CollectAndPartitionTranslationItems,
+    [] ExportName(DefId),
+    [] ContainsExternIndicator(DefId),
+    [] IsTranslatedFunction(DefId),
+    [] CodegenUnit(InternedString),
+    [] CompileCodegenUnit(InternedString),
+    [] OutputFilenames,
 );
 
 trait DepNodeParams<'a, 'gcx: 'tcx + 'a, 'tcx: 'a> : fmt::Debug {

src/librustc/lib.rs

@@ -92,6 +92,7 @@ pub mod middle {
     pub mod dependency_format;
     pub mod effect;
     pub mod entry;
+    pub mod exported_symbols;
     pub mod free_region;
     pub mod intrinsicck;
     pub mod lang_items;
@@ -103,6 +104,7 @@ pub mod middle {
     pub mod recursion_limit;
     pub mod resolve_lifetime;
     pub mod stability;
+    pub mod trans;
     pub mod weak_lang_items;
 }
 

src/librustc/middle/cstore.rs

@@ -366,8 +366,9 @@ pub trait CrateLoader {
 // In order to get this left-to-right dependency ordering, we perform a
 // topological sort of all crates putting the leaves at the right-most
 // positions.
-pub fn used_crates<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>,
-                             prefer: LinkagePreference) -> Vec<(CrateNum, LibSource)> {
+pub fn used_crates(tcx: TyCtxt, prefer: LinkagePreference)
+    -> Vec<(CrateNum, LibSource)>
+{
     let mut libs = tcx.crates()
         .iter()
         .cloned()

src/librustc/middle/exported_symbols.rs

@@ -0,0 +1,31 @@
+// Copyright 2016 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+/// The SymbolExportLevel of a symbols specifies from which kinds of crates
+/// the symbol will be exported. `C` symbols will be exported from any
+/// kind of crate, including cdylibs which export very few things.
+/// `Rust` will only be exported if the crate produced is a Rust
+/// dylib.
+#[derive(Eq, PartialEq, Debug, Copy, Clone)]
+pub enum SymbolExportLevel {
+    C,
+    Rust,
+}
+
+impl SymbolExportLevel {
+    pub fn is_below_threshold(self, threshold: SymbolExportLevel) -> bool {
+        if threshold == SymbolExportLevel::Rust {
+            // We export everything from Rust dylibs
+            true
+        } else {
+            self == SymbolExportLevel::C
+        }
+    }
+}

src/librustc/middle/reachable.rs

@@ -233,8 +233,8 @@ impl<'a, 'tcx> ReachableContext<'a, 'tcx> {
                 } else {
                     false
                 };
-                let is_extern = attr::contains_extern_indicator(&self.tcx.sess.diagnostic(),
-                                                                &item.attrs);
+                let def_id = self.tcx.hir.local_def_id(item.id);
+                let is_extern = self.tcx.contains_extern_indicator(def_id);
                 if reachable || is_extern {
                     self.reachable_symbols.insert(search_item);
                 }
@@ -369,10 +369,6 @@ impl<'a, 'tcx: 'a> ItemLikeVisitor<'tcx> for CollectPrivateImplItemsVisitor<'a,
     }
 }
 
-pub fn find_reachable<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>) -> Rc<NodeSet> {
-    tcx.reachable_set(LOCAL_CRATE)
-}
-
 fn reachable_set<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>, crate_num: CrateNum) -> Rc<NodeSet> {
     debug_assert!(crate_num == LOCAL_CRATE);
 

src/librustc/middle/trans.rs

@@ -0,0 +1,110 @@
+// Copyright 2017 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+use syntax::ast::NodeId;
+use syntax::symbol::InternedString;
+use ty::Instance;
+use util::nodemap::FxHashMap;
+
+#[derive(PartialEq, Eq, Clone, Copy, Debug, Hash)]
+pub enum TransItem<'tcx> {
+    Fn(Instance<'tcx>),
+    Static(NodeId),
+    GlobalAsm(NodeId),
+}
+
+pub struct CodegenUnit<'tcx> {
+    /// A name for this CGU. Incremental compilation requires that
+    /// name be unique amongst **all** crates.  Therefore, it should
+    /// contain something unique to this crate (e.g., a module path)
+    /// as well as the crate name and disambiguator.
+    name: InternedString,
+    items: FxHashMap<TransItem<'tcx>, (Linkage, Visibility)>,
+}
+
+#[derive(Copy, Clone, PartialEq, Eq, Hash, Debug)]
+pub enum Linkage {
+    External,
+    AvailableExternally,
+    LinkOnceAny,
+    LinkOnceODR,
+    WeakAny,
+    WeakODR,
+    Appending,
+    Internal,
+    Private,
+    ExternalWeak,
+    Common,
+}
+
+#[derive(Copy, Clone, PartialEq, Eq, Hash, Debug)]
+pub enum Visibility {
+    Default,
+    Hidden,
+    Protected,
+}
+
+impl<'tcx> CodegenUnit<'tcx> {
+    pub fn new(name: InternedString) -> CodegenUnit<'tcx> {
+        CodegenUnit {
+            name: name,
+            items: FxHashMap(),
+        }
+    }
+
+    pub fn name(&self) -> &InternedString {
+        &self.name
+    }
+
+    pub fn set_name(&mut self, name: InternedString) {
+        self.name = name;
+    }
+
+    pub fn items(&self) -> &FxHashMap<TransItem<'tcx>, (Linkage, Visibility)> {
+        &self.items
+    }
+
+    pub fn items_mut(&mut self)
+        -> &mut FxHashMap<TransItem<'tcx>, (Linkage, Visibility)>
+    {
+        &mut self.items
+    }
+}
+
+#[derive(Clone, Default)]
+pub struct Stats {
+    pub n_glues_created: usize,
+    pub n_null_glues: usize,
+    pub n_real_glues: usize,
+    pub n_fns: usize,
+    pub n_inlines: usize,
+    pub n_closures: usize,
+    pub n_llvm_insns: usize,
+    pub llvm_insns: FxHashMap<String, usize>,
+    // (ident, llvm-instructions)
+    pub fn_stats: Vec<(String, usize)>,
+}
+
+impl Stats {
+    pub fn extend(&mut self, stats: Stats) {
+        self.n_glues_created += stats.n_glues_created;
+        self.n_null_glues += stats.n_null_glues;
+        self.n_real_glues += stats.n_real_glues;
+        self.n_fns += stats.n_fns;
+        self.n_inlines += stats.n_inlines;
+        self.n_closures += stats.n_closures;
+        self.n_llvm_insns += stats.n_llvm_insns;
+
+        for (k, v) in stats.llvm_insns {
+            *self.llvm_insns.entry(k).or_insert(0) += v;
+        }
+        self.fn_stats.extend(stats.fn_stats);
+    }
+}

src/librustc/ty/context.rs

@@ -13,6 +13,7 @@
 use dep_graph::DepGraph;
 use errors::DiagnosticBuilder;
 use session::Session;
+use session::config::OutputFilenames;
 use middle;
 use hir::{TraitCandidate, HirId, ItemLocalId};
 use hir::def::{Def, Export};
@@ -64,6 +65,8 @@ use std::mem;
 use std::ops::Deref;
 use std::iter;
 use std::rc::Rc;
+use std::sync::mpsc;
+use std::sync::Arc;
 use syntax::abi;
 use syntax::ast::{self, Name, NodeId};
 use syntax::attr;
@@ -901,6 +904,16 @@ pub struct GlobalCtxt<'tcx> {
     /// error reporting, and so is lazily initialized and generally
     /// shouldn't taint the common path (hence the RefCell).
     pub all_traits: RefCell<Option<Vec<DefId>>>,
+
+    /// A general purpose channel to throw data out the back towards LLVM worker
+    /// threads.
+    ///
+    /// This is intended to only get used during the trans phase of the compiler
+    /// when satisfying the query for a particular codegen unit. Internally in
+    /// the query it'll send data along this channel to get processed later.
+    pub tx_to_llvm_workers: mpsc::Sender<Box<Any + Send>>,
+
+    output_filenames: Arc<OutputFilenames>,
 }
 
 impl<'tcx> GlobalCtxt<'tcx> {
@@ -1025,6 +1038,8 @@ impl<'a, 'gcx, 'tcx> TyCtxt<'a, 'gcx, 'tcx> {
                                   named_region_map: resolve_lifetime::NamedRegionMap,
                                   hir: hir_map::Map<'tcx>,
                                   crate_name: &str,
+                                  tx: mpsc::Sender<Box<Any + Send>>,
+                                  output_filenames: &OutputFilenames,
                                   f: F) -> R
                                   where F: for<'b> FnOnce(TyCtxt<'b, 'tcx, 'tcx>) -> R
     {
@@ -1145,6 +1160,8 @@ impl<'a, 'gcx, 'tcx> TyCtxt<'a, 'gcx, 'tcx> {
             derive_macros: RefCell::new(NodeMap()),
             stability_interner: RefCell::new(FxHashSet()),
             all_traits: RefCell::new(None),
+            tx_to_llvm_workers: tx,
+            output_filenames: Arc::new(output_filenames.clone()),
        }, f)
     }
 
@@ -2218,4 +2235,8 @@ pub fn provide(providers: &mut ty::maps::Providers) {
         assert_eq!(cnum, LOCAL_CRATE);
         Rc::new(tcx.cstore.postorder_cnums_untracked())
     };
+    providers.output_filenames = |tcx, cnum| {
+        assert_eq!(cnum, LOCAL_CRATE);
+        tcx.output_filenames.clone()
+    };
 }