rustc: Add a #[wasm_import_module] attribute

This commit adds a new attribute to the Rust compiler specific to the wasm
target (and no other targets). The `#[wasm_import_module]` attribute is used to
specify the module that a name is imported from, and is used like so:

    #[wasm_import_module = "./foo.js"]
    extern {
        fn some_js_function();
    }

Here the import of the symbol `some_js_function` is tagged with the `./foo.js`
module in the wasm output file. Wasm-the-format includes two fields on all
imports, a module and a field. The field is the symbol name (`some_js_function`
above) and the module has historically unconditionally been `"env"`. I'm not
sure if this `"env"` convention has asm.js or LLVM roots, but regardless we'd
like the ability to configure it!

The proposed ES module integration with wasm (aka a wasm module is "just another
ES module") requires that the import module of wasm imports is interpreted as an
ES module import, meaning that you'll need to encode paths, NPM packages, etc.
As a result, we'll need this to be something other than `"env"`!

Unfortunately neither our version of LLVM nor LLD supports custom import modules
(aka anything not `"env"`). My hope is that by the time LLVM 7 is released both
will have support, but in the meantime this commit adds some primitive
encoding/decoding of wasm files to the compiler. This way rustc postprocesses
the wasm module that LLVM emits to ensure it's got all the imports we'd like to
have in it.

Eventually I'd ideally like to unconditionally require this attribute to be
placed on all `extern { ... }` blocks. For now though it seemed prudent to add
it as an unstable attribute, so for now it's not required (as that'd force usage
of a feature gate). Hopefully it doesn't take too long to "stabilize" this!

cc rust-lang-nursery/rust-wasm#29

src/ci/docker/wasm32-unknown/Dockerfile

@@ -26,6 +26,7 @@ ENV RUST_CONFIGURE_ARGS \
   --set rust.lld
 
 ENV SCRIPT python2.7 /checkout/x.py test --target $TARGETS \
+  src/test/run-make \
   src/test/ui \
   src/test/run-pass \
   src/test/compile-fail \

src/librustc/dep_graph/dep_node.rs

@@ -593,6 +593,7 @@ define_dep_nodes!( <'tcx>
     [] ImplementationsOfTrait { krate: CrateNum, trait_id: DefId },
     [] AllTraitImplementations(CrateNum),
 
+    [] DllimportForeignItems(CrateNum),
     [] IsDllimportForeignItem(DefId),
     [] IsStaticallyIncludedForeignItem(DefId),
     [] NativeLibraryKind(DefId),
@@ -655,6 +656,8 @@ define_dep_nodes!( <'tcx>
     [input] Features,
 
     [] ProgramClausesFor(DefId),
+    [] WasmImportModuleMap(CrateNum),
+    [] ForeignModules(CrateNum),
 );
 
 trait DepNodeParams<'a, 'gcx: 'tcx + 'a, 'tcx: 'a> : fmt::Debug {

src/librustc/hir/check_attr.rs

@@ -26,6 +26,7 @@ enum Target {
     Union,
     Enum,
     Const,
+    ForeignMod,
     Other,
 }
 
@@ -37,6 +38,7 @@ impl Target {
             hir::ItemUnion(..) => Target::Union,
             hir::ItemEnum(..) => Target::Enum,
             hir::ItemConst(..) => Target::Const,
+            hir::ItemForeignMod(..) => Target::ForeignMod,
             _ => Target::Other,
         }
     }
@@ -57,25 +59,42 @@ impl<'a, 'tcx> CheckAttrVisitor<'a, 'tcx> {
                 .emit();
         }
 
+        let mut has_wasm_import_module = false;
         for attr in &item.attrs {
-            if let Some(name) = attr.name() {
-                if name == "inline" {
-                    self.check_inline(attr, item, target)
+            if attr.check_name("inline") {
+                self.check_inline(attr, item, target)
+            } else if attr.check_name("wasm_import_module") {
+                has_wasm_import_module = true;
+                if attr.value_str().is_none() {
+                    self.tcx.sess.span_err(attr.span, "\
+                        must be of the form #[wasm_import_module = \"...\"]");
+                }
+                if target != Target::ForeignMod {
+                    self.tcx.sess.span_err(attr.span, "\
+                        must only be attached to foreign modules");
+                }
+            } else if attr.check_name("wasm_custom_section") {
+                if target != Target::Const {
+                    self.tcx.sess.span_err(attr.span, "only allowed on consts");
                 }
 
-                if name == "wasm_custom_section" {
-                    if target != Target::Const {
-                        self.tcx.sess.span_err(attr.span, "only allowed on consts");
-                    }
-
-                    if attr.value_str().is_none() {
-                        self.tcx.sess.span_err(attr.span, "must be of the form \
-                            #[wasm_custom_section = \"foo\"]");
-                    }
+                if attr.value_str().is_none() {
+                    self.tcx.sess.span_err(attr.span, "must be of the form \
+                        #[wasm_custom_section = \"foo\"]");
                 }
             }
         }
 
+        if target == Target::ForeignMod &&
+            !has_wasm_import_module &&
+            self.tcx.sess.target.target.arch == "wasm32" &&
+            false // FIXME: eventually enable this warning when stable
+        {
+            self.tcx.sess.span_warn(item.span, "\
+                must have a #[wasm_import_module = \"...\"] attribute, this \
+                will become a hard error before too long");
+        }
+
         self.check_repr(item, target);
     }
 

src/librustc/ich/impls_cstore.rs

@@ -33,7 +33,12 @@ impl_stable_hash_for!(struct middle::cstore::NativeLibrary {
     kind,
     name,
     cfg,
-    foreign_items
+    foreign_module
+});
+
+impl_stable_hash_for!(struct middle::cstore::ForeignModule {
+    foreign_items,
+    def_id
 });
 
 impl_stable_hash_for!(enum middle::cstore::LinkagePreference {

src/librustc/middle/cstore.rs

@@ -132,7 +132,13 @@ pub struct NativeLibrary {
     pub kind: NativeLibraryKind,
     pub name: Symbol,
     pub cfg: Option<ast::MetaItem>,
+    pub foreign_module: Option<DefId>,
+}
+
+#[derive(Clone, Hash, RustcEncodable, RustcDecodable)]
+pub struct ForeignModule {
     pub foreign_items: Vec<DefId>,
+    pub def_id: DefId,
 }
 
 pub enum LoadedMacro {

src/librustc/ty/maps/config.rs

@@ -430,6 +430,12 @@ impl<'tcx> QueryDescription<'tcx> for queries::native_libraries<'tcx> {
     }
 }
 
+impl<'tcx> QueryDescription<'tcx> for queries::foreign_modules<'tcx> {
+    fn describe(_tcx: TyCtxt, _: CrateNum) -> String {
+        format!("looking up the foreign modules of a linked crate")
+    }
+}
+
 impl<'tcx> QueryDescription<'tcx> for queries::plugin_registrar_fn<'tcx> {
     fn describe(_tcx: TyCtxt, _: CrateNum) -> String {
         format!("looking up the plugin registrar for a crate")
@@ -705,6 +711,18 @@ impl<'tcx> QueryDescription<'tcx> for queries::program_clauses_for<'tcx> {
     }
 }
 
+impl<'tcx> QueryDescription<'tcx> for queries::wasm_import_module_map<'tcx> {
+    fn describe(_tcx: TyCtxt, _: CrateNum) -> String {
+        format!("wasm import module map")
+    }
+}
+
+impl<'tcx> QueryDescription<'tcx> for queries::dllimport_foreign_items<'tcx> {
+    fn describe(_tcx: TyCtxt, _: CrateNum) -> String {
+        format!("wasm import module map")
+    }
+}
+
 macro_rules! impl_disk_cacheable_query(
     ($query_name:ident, |$key:tt| $cond:expr) => {
         impl<'tcx> QueryDescription<'tcx> for queries::$query_name<'tcx> {

src/librustc/ty/maps/mod.rs

@@ -18,7 +18,7 @@ use infer::canonical::{Canonical, QueryResult};
 use lint;
 use middle::borrowck::BorrowCheckResult;
 use middle::cstore::{ExternCrate, LinkagePreference, NativeLibrary,
-                     ExternBodyNestedBodies};
+                     ExternBodyNestedBodies, ForeignModule};
 use middle::cstore::{NativeLibraryKind, DepKind, CrateSource, ExternConstBody};
 use middle::privacy::AccessLevels;
 use middle::reachable::ReachableSet;
@@ -320,6 +320,9 @@ define_maps! { <'tcx>
 
 
     [] fn native_libraries: NativeLibraries(CrateNum) -> Lrc<Vec<NativeLibrary>>,
+
+    [] fn foreign_modules: ForeignModules(CrateNum) -> Lrc<Vec<ForeignModule>>,
+
     [] fn plugin_registrar_fn: PluginRegistrarFn(CrateNum) -> Option<DefId>,
     [] fn derive_registrar_fn: DeriveRegistrarFn(CrateNum) -> Option<DefId>,
     [] fn crate_disambiguator: CrateDisambiguator(CrateNum) -> CrateDisambiguator,
@@ -331,6 +334,8 @@ define_maps! { <'tcx>
     [] fn all_trait_implementations: AllTraitImplementations(CrateNum)
         -> Lrc<Vec<DefId>>,
 
+    [] fn dllimport_foreign_items: DllimportForeignItems(CrateNum)
+        -> Lrc<FxHashSet<DefId>>,
     [] fn is_dllimport_foreign_item: IsDllimportForeignItem(DefId) -> bool,
     [] fn is_statically_included_foreign_item: IsStaticallyIncludedForeignItem(DefId) -> bool,
     [] fn native_library_kind: NativeLibraryKind(DefId)
@@ -426,6 +431,8 @@ define_maps! { <'tcx>
     [] fn program_clauses_for: ProgramClausesFor(DefId) -> Lrc<Vec<Clause<'tcx>>>,
 
     [] fn wasm_custom_sections: WasmCustomSections(CrateNum) -> Lrc<Vec<DefId>>,
+    [] fn wasm_import_module_map: WasmImportModuleMap(CrateNum)
+        -> Lrc<FxHashMap<DefId, String>>,
 }
 
 //////////////////////////////////////////////////////////////////////

src/librustc/ty/maps/plumbing.rs

@@ -886,6 +886,9 @@ pub fn force_from_dep_node<'a, 'gcx, 'lcx>(tcx: TyCtxt<'a, 'gcx, 'lcx>,
             force!(all_trait_implementations, krate!());
         }
 
+        DepKind::DllimportForeignItems => {
+            force!(dllimport_foreign_items, krate!());
+        }
         DepKind::IsDllimportForeignItem => {
             force!(is_dllimport_foreign_item, def_id!());
         }
@@ -941,6 +944,8 @@ pub fn force_from_dep_node<'a, 'gcx, 'lcx>(tcx: TyCtxt<'a, 'gcx, 'lcx>,
 
         DepKind::ProgramClausesFor => { force!(program_clauses_for, def_id!()); }
         DepKind::WasmCustomSections => { force!(wasm_custom_sections, krate!()); }
+        DepKind::WasmImportModuleMap => { force!(wasm_import_module_map, krate!()); }
+        DepKind::ForeignModules => { force!(foreign_modules, krate!()); }
     }
 
     true

src/librustc_metadata/creader.rs

@@ -12,7 +12,6 @@
 
 use cstore::{self, CStore, CrateSource, MetadataBlob};
 use locator::{self, CratePaths};
-use native_libs::relevant_lib;
 use schema::CrateRoot;
 use rustc_data_structures::sync::{Lrc, RwLock, Lock};
 
@@ -230,7 +229,7 @@ impl<'a> CrateLoader<'a> {
             .map(|trait_impls| (trait_impls.trait_id, trait_impls.impls))
             .collect();
 
-        let mut cmeta = cstore::CrateMetadata {
+        let cmeta = cstore::CrateMetadata {
             name,
             extern_crate: Lock::new(None),
             def_path_table: Lrc::new(def_path_table),
@@ -250,25 +249,8 @@ impl<'a> CrateLoader<'a> {
                 rlib,
                 rmeta,
             },
-            // Initialize this with an empty set. The field is populated below
-            // after we were able to deserialize its contents.
-            dllimport_foreign_items: FxHashSet(),
         };
 
-        let dllimports: FxHashSet<_> = cmeta
-            .root
-            .native_libraries
-            .decode((&cmeta, self.sess))
-            .filter(|lib| relevant_lib(self.sess, lib) &&
-                          lib.kind == cstore::NativeLibraryKind::NativeUnknown)
-            .flat_map(|lib| {
-                assert!(lib.foreign_items.iter().all(|def_id| def_id.krate == cnum));
-                lib.foreign_items.into_iter().map(|def_id| def_id.index)
-            })
-            .collect();
-
-        cmeta.dllimport_foreign_items = dllimports;
-
         let cmeta = Lrc::new(cmeta);
         self.cstore.set_crate_data(cnum, cmeta.clone());
         (cnum, cmeta)

src/librustc_metadata/cstore.rs

@@ -20,7 +20,7 @@ use rustc::middle::cstore::{DepKind, ExternCrate, MetadataLoader};
 use rustc::session::{Session, CrateDisambiguator};
 use rustc_back::PanicStrategy;
 use rustc_data_structures::indexed_vec::IndexVec;
-use rustc::util::nodemap::{FxHashMap, FxHashSet, NodeMap};
+use rustc::util::nodemap::{FxHashMap, NodeMap};
 
 use rustc_data_structures::sync::{Lrc, RwLock, Lock};
 use syntax::{ast, attr};
@@ -30,7 +30,7 @@ use syntax_pos;
 
 pub use rustc::middle::cstore::{NativeLibrary, NativeLibraryKind, LinkagePreference};
 pub use rustc::middle::cstore::NativeLibraryKind::*;
-pub use rustc::middle::cstore::{CrateSource, LibSource};
+pub use rustc::middle::cstore::{CrateSource, LibSource, ForeignModule};
 
 pub use cstore_impl::{provide, provide_extern};
 
@@ -84,8 +84,6 @@ pub struct CrateMetadata {
     pub source: CrateSource,
 
     pub proc_macros: Option<Vec<(ast::Name, Lrc<SyntaxExtension>)>>,
-    // Foreign items imported from a dylib (Windows only)
-    pub dllimport_foreign_items: FxHashSet<DefIndex>,
 }
 
 pub struct CStore {

src/librustc_metadata/cstore_impl.rs

@@ -12,6 +12,7 @@ use cstore;
 use encoder;
 use link_args;
 use native_libs;
+use foreign_modules;
 use schema;
 
 use rustc::ty::maps::QueryConfig;
@@ -197,6 +198,7 @@ provide! { <'tcx> tcx, def_id, other, cdata,
         Lrc::new(reachable_non_generics)
     }
     native_libraries => { Lrc::new(cdata.get_native_libraries(tcx.sess)) }
+    foreign_modules => { Lrc::new(cdata.get_foreign_modules(tcx.sess)) }
     plugin_registrar_fn => {
         cdata.root.plugin_registrar_fn.map(|index| {
             DefId { krate: def_id.krate, index }
@@ -224,9 +226,6 @@ provide! { <'tcx> tcx, def_id, other, cdata,
         Lrc::new(result)
     }
 
-    is_dllimport_foreign_item => {
-        cdata.is_dllimport_foreign_item(def_id.index)
-    }
     visibility => { cdata.get_visibility(def_id.index) }
     dep_kind => {
         let r = *cdata.dep_kind.lock();
@@ -306,13 +305,28 @@ pub fn provide<'tcx>(providers: &mut Providers<'tcx>) {
             tcx.native_libraries(id.krate)
                 .iter()
                 .filter(|lib| native_libs::relevant_lib(&tcx.sess, lib))
-                .find(|l| l.foreign_items.contains(&id))
+                .find(|lib| {
+                    let fm_id = match lib.foreign_module {
+                        Some(id) => id,
+                        None => return false,
+                    };
+                    tcx.foreign_modules(id.krate)
+                        .iter()
+                        .find(|m| m.def_id == fm_id)
+                        .expect("failed to find foreign module")
+                        .foreign_items
+                        .contains(&id)
+                })
                 .map(|l| l.kind)
         },
         native_libraries: |tcx, cnum| {
             assert_eq!(cnum, LOCAL_CRATE);
             Lrc::new(native_libs::collect(tcx))
         },
+        foreign_modules: |tcx, cnum| {
+            assert_eq!(cnum, LOCAL_CRATE);
+            Lrc::new(foreign_modules::collect(tcx))
+        },
         link_args: |tcx, cnum| {
             assert_eq!(cnum, LOCAL_CRATE);
             Lrc::new(link_args::collect(tcx))

src/librustc_metadata/decoder.rs

@@ -10,7 +10,7 @@
 
 // Decoding metadata from a single crate's metadata
 
-use cstore::{self, CrateMetadata, MetadataBlob, NativeLibrary};
+use cstore::{self, CrateMetadata, MetadataBlob, NativeLibrary, ForeignModule};
 use schema::*;
 
 use rustc_data_structures::sync::{Lrc, ReadGuard};
@@ -1031,6 +1031,10 @@ impl<'a, 'tcx> CrateMetadata {
         self.root.native_libraries.decode((self, sess)).collect()
     }
 
+    pub fn get_foreign_modules(&self, sess: &Session) -> Vec<ForeignModule> {
+        self.root.foreign_modules.decode((self, sess)).collect()
+    }
+
     pub fn get_dylib_dependency_formats(&self) -> Vec<(CrateNum, LinkagePreference)> {
         self.root
             .dylib_dependency_formats
@@ -1103,10 +1107,6 @@ impl<'a, 'tcx> CrateMetadata {
         }
     }
 
-    pub fn is_dllimport_foreign_item(&self, id: DefIndex) -> bool {
-        self.dllimport_foreign_items.contains(&id)
-    }
-
     pub fn fn_sig(&self,
                   id: DefIndex,
                   tcx: TyCtxt<'a, 'tcx, 'tcx>)