rustc: Add knowledge of Windows subsystems.

This commit is an implementation of [RFC 1665] which adds support for the
`#![windows_subsystem]` attribute. This attribute allows specifying either the
"windows" or "console" subsystems on Windows to the linker.

[RFC 1665]: https://github.com/rust-lang/rfcs/blob/master/text/1665-windows-subsystem.md

Previously all Rust executables were compiled as the "console" subsystem which
meant that if you wanted a graphical application it would erroneously pop up a
console whenever opened. When compiling an application, however, this is
undesired behavior and the "windows" subsystem is used instead to have control
over user interactions.

This attribute is validated, but ignored on all non-Windows platforms.

cc #37499

Author: alexcrichton

src/librustc_trans/back/link.rs

@@ -636,7 +636,7 @@ fn link_natively(sess: &Session,
     {
         let mut linker = trans.linker_info.to_linker(&mut cmd, &sess);
         link_args(&mut *linker, sess, crate_type, tmpdir,
-                  objects, out_filename, outputs);
+                  objects, out_filename, outputs, trans);
     }
     cmd.args(&sess.target.target.options.late_link_args);
     for obj in &sess.target.target.options.post_link_objects {
@@ -711,7 +711,8 @@ fn link_args(cmd: &mut Linker,
              tmpdir: &Path,
              objects: &[PathBuf],
              out_filename: &Path,
-             outputs: &OutputFilenames) {
+             outputs: &OutputFilenames,
+             trans: &CrateTranslation) {
 
     // The default library location, we need this to find the runtime.
     // The location of crates will be determined as needed.
@@ -726,6 +727,13 @@ fn link_args(cmd: &mut Linker,
     }
     cmd.output_filename(out_filename);
 
+    if crate_type == config::CrateTypeExecutable &&
+       sess.target.target.options.is_like_windows {
+        if let Some(ref s) = trans.windows_subsystem {
+            cmd.subsystem(s);
+        }
+    }
+
     // If we're building a dynamic library then some platforms need to make sure
     // that all symbols are exported correctly from the dynamic library.
     if crate_type != config::CrateTypeExecutable {

src/librustc_trans/back/linker.rs

@@ -92,6 +92,7 @@ pub trait Linker {
     fn whole_archives(&mut self);
     fn no_whole_archives(&mut self);
     fn export_symbols(&mut self, tmpdir: &Path, crate_type: CrateType);
+    fn subsystem(&mut self, subsystem: &str);
 }
 
 pub struct GnuLinker<'a> {
@@ -294,6 +295,10 @@ impl<'a> Linker for GnuLinker<'a> {
 
         self.cmd.arg(arg);
     }
+
+    fn subsystem(&mut self, subsystem: &str) {
+        self.cmd.arg(&format!("-Wl,--subsystem,{}", subsystem));
+    }
 }
 
 pub struct MsvcLinker<'a> {
@@ -441,6 +446,30 @@ impl<'a> Linker for MsvcLinker<'a> {
         arg.push(path);
         self.cmd.arg(&arg);
     }
+
+    fn subsystem(&mut self, subsystem: &str) {
+        // Note that previous passes of the compiler validated this subsystem,
+        // so we just blindly pass it to the linker.
+        self.cmd.arg(&format!("/SUBSYSTEM:{}", subsystem));
+
+        // Windows has two subsystems we're interested in right now, the console
+        // and windows subsystems. These both implicitly have different entry
+        // points (starting symbols). The console entry point starts with
+        // `mainCRTStartup` and the windows entry point starts with
+        // `WinMainCRTStartup`. These entry points, defined in system libraries,
+        // will then later probe for either `main` or `WinMain`, respectively to
+        // start the application.
+        //
+        // In Rust we just always generate a `main` function so we want control
+        // to always start there, so we force the entry point on the windows
+        // subsystem to be `mainCRTStartup` to get everything booted up
+        // correctly.
+        //
+        // For more information see RFC #1665
+        if subsystem == "windows" {
+            self.cmd.arg("/ENTRY:mainCRTStartup");
+        }
+    }
 }
 
 fn exported_symbols(scx: &SharedCrateContext,

src/librustc_trans/base.rs

@@ -1611,7 +1611,8 @@ pub fn trans_crate<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>,
             metadata: metadata,
             reachable: vec![],
             no_builtins: no_builtins,
-            linker_info: linker_info
+            linker_info: linker_info,
+            windows_subsystem: None,
         };
     }
 
@@ -1747,14 +1748,26 @@ pub fn trans_crate<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>,
 
     let linker_info = LinkerInfo::new(&shared_ccx, &reachable_symbols);
 
+    let subsystem = attr::first_attr_value_str_by_name(&krate.attrs,
+                                                       "windows_subsystem");
+    let windows_subsystem = subsystem.map(|subsystem| {
+        if subsystem != "windows" && subsystem != "console" {
+            tcx.sess.fatal(&format!("invalid windows subsystem `{}`, only \
+                                     `windows` and `console` are allowed",
+                                    subsystem));
+        }
+        subsystem.to_string()
+    });
+
     CrateTranslation {
         modules: modules,
         metadata_module: metadata_module,
         link: link_meta,
         metadata: metadata,
         reachable: reachable_symbols,
         no_builtins: no_builtins,
-        linker_info: linker_info
+        linker_info: linker_info,
+        windows_subsystem: windows_subsystem,
     }
 }
 

src/librustc_trans/lib.rs

@@ -169,6 +169,7 @@ pub struct CrateTranslation {
     pub metadata: Vec<u8>,
     pub reachable: Vec<String>,
     pub no_builtins: bool,
+    pub windows_subsystem: Option<String>,
     pub linker_info: back::linker::LinkerInfo
 }
 

src/libsyntax/feature_gate.rs

@@ -309,6 +309,9 @@ declare_features! (
 
     // Allows field shorthands (`x` meaning `x: x`) in struct literal expressions.
     (active, field_init_shorthand, "1.14.0", Some(37340)),
+
+    // The #![windows_subsystem] attribute
+    (active, windows_subsystem, "1.14.0", Some(37499)),
 );
 
 declare_features! (
@@ -713,6 +716,12 @@ pub const KNOWN_ATTRIBUTES: &'static [(&'static str, AttributeType, AttributeGat
                                               "defining reflective traits is still evolving",
                                               cfg_fn!(reflect))),
 
+    ("windows_subsystem", Whitelisted, Gated(Stability::Unstable,
+                                             "windows_subsystem",
+                                             "the windows subsystem attribute \
+                                              id currently unstable",
+                                             cfg_fn!(windows_subsystem))),
+
     // Crate level attributes
     ("crate_name", CrateLevel, Ungated),
     ("crate_type", CrateLevel, Ungated),

src/test/compile-fail/windows-subsystem-gated.rs

@@ -0,0 +1,14 @@
+// Copyright 2016 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+#![windows_subsystem = "console"]
+//~^ ERROR: the windows subsystem attribute is currently unstable
+
+fn main() {}

src/test/compile-fail/windows-subsystem-invalid.rs

@@ -0,0 +1,15 @@
+// Copyright 2016 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+#![feature(windows_subsystem)]
+#![windows_subsystem = "wrong"]
+//~^ ERROR: invalid subsystem `wrong`, only `windows` and `console` are allowed
+
+fn main() {}

src/test/run-make/windows-subsystem/Makefile

@@ -0,0 +1,5 @@
+-include ../tools.mk
+
+all:
+	$(RUSTC) windows.rs
+	$(RUSTC) console.rs

src/test/run-make/windows-subsystem/console.rs

@@ -0,0 +1,15 @@
+// Copyright 2016 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+#![feature(windows_subsystem)]
+#![windows_subsystem = "console"]
+
+fn main() {}
+

src/test/run-make/windows-subsystem/windows.rs

@@ -0,0 +1,14 @@
+// Copyright 2016 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+#![feature(windows_subsystem)]
+#![windows_subsystem = "windows"]
+
+fn main() {}