library-ifying of tools - turning cargo, rustdoc, fuzzer, into libraries, as per #3543

AUTHORS.txt

@@ -9,9 +9,11 @@ Aleksander Balicki <balicki.aleksander@gmail.com>
 Alex Rønne Petersen <alex@lycus.org>
 Alexander Stavonin <a.stavonin@gmail.com>
 Andreas Gal <gal@mozilla.com>
+Andrew Paseltiner <apaseltiner@gmail.com>
 Arkaitz Jimenez <arkaitzj@gmail.com>
 Armin Ronacher <armin.ronacher@active-4.com>
 Austin Seipp <mad.one@gmail.com>
+auREAX <mark@xn--hwg34fba.ws>
 Ben Blum <bblum@andrew.cmu.edu>
 Ben Striegel <ben.striegel@gmail.com>
 Benjamin Herr <ben@0x539.de>
@@ -39,13 +41,16 @@ Erick Tryzelaar <erick.tryzelaar@gmail.com>
 Erik Rose <erik@mozilla.com>
 Evan McClanahan <evan@evanmcc.com>
 Francisco Souza <f@souza.cc>
+Gabriel <g2p.code@gmail.com>
 Gareth Daniel Smith <garethdanielsmith@gmail.com>
 Glenn Willen <gwillen@nerdnet.org>
+Gonçalo Cabrita <_@gmcabrita.com>
 Graham Fawcett <fawcett@uwindsor.ca>
 Grahame Bowland <grahame@angrygoats.net>
 Haitao Li <lihaitao@gmail.com>
 Ian D. Bollinger <ian.bollinger@gmail.com>
 Ivano Coppola <rgbfirefox@gmail.com>
+Jacob Harris Cryer Kragh <jhckragh@gmail.com>
 Jacob Parker <j3parker@csclub.uwaterloo.ca>
 Jason Orendorff <jorendorff@mozilla.com>
 Jed Davis <jld@panix.com>
@@ -68,7 +73,9 @@ Kevin Cantu <me@kevincantu.org>
 Lennart Kudling
 Lindsey Kuper <lindsey@rockstargirl.org>
 Luca Bruno <lucab@debian.org>
+Luqman Aden <laden@csclub.uwaterloo.ca>
 Magnus Auvinen <magnus.auvinen@gmail.com>
+Mahmut Bulut <mahmutbulut0@gmail.com>
 Margaret Meyerhofer <mmeyerho@andrew.cmu.edu>
 Marijn Haverbeke <marijnh@gmail.com>
 Matt Brubeck <mbrubeck@limpet.net>
@@ -103,4 +110,5 @@ Tomoki Aonuma <uasi@99cm.org>
 Tycho Sci <tychosci@gmail.com>
 Vincent Belliard <vincent@famillebelliard.fr>
 Wade Mealing <wmealing@gmail.com>
+Yasuhiro Fujii <y-fujii@mimosa-pudica.net>
 Zack Corr <zackcorr95@gmail.com>

Makefile.in

@@ -120,19 +120,28 @@ CFG_CORELIB :=$(call CFG_LIB_NAME,core)
 CFG_STDLIB :=$(call CFG_LIB_NAME,std)
 CFG_LIBRUSTC :=$(call CFG_LIB_NAME,rustc)
 CFG_LIBSYNTAX :=$(call CFG_LIB_NAME,syntax)
+CFG_LIBFUZZER :=$(call CFG_LIB_NAME,fuzzer)
+CFG_LIBCARGO :=$(call CFG_LIB_NAME,cargo)
+CFG_LIBRUSTDOC :=$(call CFG_LIB_NAME,rustdoc)
 
 STDLIB_GLOB :=$(call CFG_LIB_GLOB,std)
 CORELIB_GLOB :=$(call CFG_LIB_GLOB,core)
 LIBRUSTC_GLOB :=$(call CFG_LIB_GLOB,rustc)
 LIBSYNTAX_GLOB :=$(call CFG_LIB_GLOB,syntax)
+LIBFUZZER_GLOB :=$(call CFG_LIB_GLOB,fuzzer)
+LIBCARGO_GLOB :=$(call CFG_LIB_GLOB,cargo)
+LIBRUSTDOC_GLOB :=$(call CFG_LIB_GLOB,rustdoc)
 STDLIB_DSYM_GLOB :=$(call CFG_LIB_DSYM_GLOB,std)
 CORELIB_DSYM_GLOB :=$(call CFG_LIB_DSYM_GLOB,core)
 LIBRUSTC_DSYM_GLOB :=$(call CFG_LIB_DSYM_GLOB,rustc)
 LIBSYNTAX_DSYM_GLOB :=$(call CFG_LIB_DSYM_GLOB,syntax)
+LIBFUZZER_DSYM_GLOB :=$(call CFG_LIB_DSYM_GLOB,fuzzer)
+LIBCARGO_DSYM_GLOB :=$(call CFG_LIB_DSYM_GLOB,cargo)
+LIBRUSTDOC_DSYM_GLOB :=$(call CFG_LIB_DSYM_GLOB,rustdoc)
 
 # version-string calculation
 CFG_GIT_DIR := $(CFG_SRC_DIR).git
-CFG_RELEASE = 0.4
+CFG_RELEASE = 0.5
 CFG_VERSION = $(CFG_RELEASE)
 
 ifneq ($(wildcard $(CFG_GIT)),)
@@ -144,8 +153,9 @@ ifneq ($(wildcard $(CFG_GIT_DIR)),)
 endif
 endif
 
-ifdef CFG_DISABLE_VALGRIND
-  $(info cfg: disabling valgrind (CFG_DISABLE_VALGRIND))
+ifdef CFG_ENABLE_VALGRIND
+  $(info cfg: enabling valgrind (CFG_ENABLE_VALGRIND))
+else
   CFG_VALGRIND :=
 endif
 ifdef CFG_BAD_VALGRIND

README.md

@@ -6,52 +6,61 @@ documentation.
 
 ## Installation
 
-The Rust compiler is slightly unusual in that it is written in Rust and
-therefore must be built by a precompiled "snapshot" version of itself (made in
-an earlier state of development). As such, source builds require that:
+The Rust compiler currently must be built from a [tarball], unless you
+are on Windows, in which case using the [installer][win-exe] is
+recommended.
 
-* You are connected to the internet, to fetch snapshots.
+Since the Rust compiler is written in Rust, it must be built by
+a precompiled "snapshot" version of itself (made in an earlier state
+of development). As such, source builds require a connection to
+the Internet, to fetch snapshots, and an OS that can execute the
+available snapshot binaries.
 
-* You can at least execute snapshot binaries of one of the forms we offer
-  them in. Currently we build and test snapshots on:
+Snapshot binaries are currently built and tested on several platforms:
 
-  * Windows (7, server 2008 r2) x86 only
-  * Linux 2.6.x (various distributions) x86 and x86-64
-  * OSX 10.6 ("Snow Leopard") or 10.7 ("Lion") x86 and x86-64
+* Windows (7, Server 2008 R2), x86 only
+* Linux (various distributions), x86 and x86-64
+* OSX 10.6 ("Snow Leopard") or greater, x86 and x86-64
 
-You may find other platforms work, but these are our "tier 1" supported build
-environments that are most likely to work. Further platforms will be added to
-the list in the future via cross-compilation.
+You may find that other platforms work, but these are our "tier 1"
+supported build environments that are most likely to work.
 
-To build from source you will also need the following prerequisite packages:
+> ***Note:*** Windows users should read the detailed
+> [getting started][wiki-start] notes on the wiki. Even when using
+> the binary installer the Windows build requires a MinGW installation,
+> the precise details of which are not discussed here.
+
+To build from source you will also need the following prerequisite
+packages:
 
 * g++ 4.4 or clang++ 3.x
-* python 2.6 or later
+* python 2.6 or later (but not 3.x)
 * perl 5.0 or later
 * gnu make 3.81 or later
 * curl
 
-Assuming you're on a relatively modern Linux/OSX system and have met the
-prerequisites, something along these lines should work:
+Assuming you're on a relatively modern *nix system and have met the
+prerequisites, something along these lines should work.
 
+    $ wget http://dl.rust-lang.org/dist/rust-0.4.tar.gz
     $ tar -xzf rust-0.4.tar.gz
     $ cd rust-0.4
     $ ./configure
     $ make && make install
 
-When complete, make install will place the following programs into
-/usr/local/bin:
-
-* rustc, the Rust compiler
-* rustdoc, the API-documentation tool
-* cargo, the Rust package manager
+You may need to use `sudo make install` if you do not normally have
+permission to modify the destination directory. The install locations
+can be adjusted by passing a `--prefix` argument to
+`configure`. Various other options are also supported, pass `--help`
+for more information on them.
 
-In addition to a manual page under /usr/local/share/man and a set of host and
-target libraries under /usr/local/lib/rustc.
+When complete, `make install` will place several programs into
+`/usr/local/bin`: `rustc`, the Rust compiler; `rustdoc`, the
+API-documentation tool, and `cargo`, the Rust package manager.
 
-The install locations can be adjusted by passing a --prefix argument to
-configure. Various other options are also supported, pass --help for more
-information on them.
+[wiki-start]: https://github.com/mozilla/rust/wiki/Note-getting-started-developing-Rust
+[tarball]: http://dl.rust-lang.org/dist/rust-0.4.tar.gz
+[win-exe]: http://dl.rust-lang.org/dist/rust-0.4-install.exe
 
 
 ## License

RELEASES.txt

@@ -1,7 +1,7 @@
 Version 0.4 (October 2012)
 --------------------------
 
-   * ~1500 changes, numerous bugfixes
+   * ~2000 changes, numerous bugfixes
 
    * Syntax
       * All keywords are now strict and may not be used as identifiers anywhere

configure

@@ -187,6 +187,7 @@ need_cmd cmp
 need_cmd mkdir
 need_cmd printf
 need_cmd cut
+need_cmd head
 need_cmd grep
 need_cmd xargs
 need_cmd cp
@@ -257,6 +258,16 @@ case $CFG_CPUTYPE in
         err "unknown CPU type: $CFG_CPUTYPE"
 esac
 
+# Detect 64 bit linux systems with 32 bit userland and force 32 bit compilation
+if [ $CFG_OSTYPE = unknown-linux-gnu -a $CFG_CPUTYPE = x86_64 ]
+then
+    file -L "$SHELL" | grep -q "x86[_-]64"
+    if [ $? != 0 ]; then
+        CFG_CPUTYPE=i686
+    fi
+fi
+
+
 DEFAULT_HOST_TRIPLE="${CFG_CPUTYPE}-${CFG_OSTYPE}"
 
 CFG_SRC_DIR="$(cd $(dirname $0) && pwd)/"
@@ -283,7 +294,7 @@ else
 fi
 
 opt sharedstd 1 "build libstd as a shared library"
-opt valgrind 1 "run tests with valgrind (memcheck by default)"
+opt valgrind 0 "run tests with valgrind (memcheck by default)"
 opt helgrind 0 "run tests with helgrind instead of memcheck"
 opt docs     1 "build documentation"
 opt optimize 1 "build optimized rust code"
@@ -295,6 +306,7 @@ opt manage-submodules 1 "let the build manage the git submodules"
 opt mingw-cross 0 "cross-compile for win32 using mingw"
 opt clang 0 "prefer clang to gcc for building the runtime"
 opt local-rust 0 "use an installed rustc rather than downloading a snapshot"
+opt pax-flags 0 "apply PaX flags to rustc binaries (required for GRSecurity/PaX-patched kernels)"
 valopt prefix "/usr/local" "set installation prefix"
 valopt local-rust-root "/usr/local" "set prefix for local rust binary"
 valopt llvm-root "" "set LLVM root"
@@ -343,14 +355,69 @@ probe CFG_PDFLATEX         pdflatex
 probe CFG_XETEX            xetex
 probe CFG_LUATEX           luatex
 probe CFG_NODE             nodejs node
+if [ "$CFG_OSTYPE" = "unknown-linux-gnu" ]
+then
+    probe CFG_PAXCTL           paxctl /sbin/paxctl
+    probe CFG_ZCAT             zcat
+fi
 
 if [ ! -z "$CFG_PANDOC" ]
 then
-    PV=$(pandoc --version | awk '/^pandoc/ {print $2}')
-    if [ "$PV" \< "1.8" ]
+    PANDOC_VER_LINE=$(pandoc --version | grep '^pandoc ')
+    PANDOC_VER=${PANDOC_VER_LINE#pandoc }
+    PV_MAJOR_MINOR=${PANDOC_VER%.[0-9]*}
+    PV_MAJOR=${PV_MAJOR_MINOR%%[.][0-9]*}
+    PV_MINOR=${PV_MAJOR_MINOR#[0-9]*[.]}
+    PV_MINOR=${PV_MINOR%%[.][0-9]*}
+    if [ "$PV_MAJOR" -lt "1" ] || [ "$PV_MINOR" -lt "8" ]
     then
-	step_msg "pandoc $PV is too old. disabling"
-	BAD_PANDOC=1
+		step_msg "pandoc $PV_MAJOR.$PV_MINOR is too old. disabling"
+		BAD_PANDOC=1
+    fi
+fi
+
+if [ "$CFG_OSTYPE" = "unknown-linux-gnu" ]
+then
+    if [ ! -z "$CFG_ENABLE_PAX_FLAGS" -a -z "$CFG_PAXCTL" ]
+    then
+        err "enabled PaX markings but no paxctl binary found"
+    fi
+
+    if [ -z "$CFG_DISABLE_PAX_FLAGS" ]
+    then
+        # GRSecurity/PaX detection. This can be very flaky.
+        GRSEC_DETECTED=
+
+        # /dev/grsec only exists if CONFIG_GRKERNSEC_NO_RBAC is not set.
+        # /proc/sys/kernel/grsecurity is not available if ÇONFIG_GRKERNSEC_SYSCTL is not set.
+        if [ -e /dev/grsec -o -d /proc/sys/kernel/grsecurity ]
+        then
+            GRSEC_DETECTED=1
+        # /proc/config.gz is normally only available to root, and only if CONFIG_IKCONFIG_PROC has been set.
+        elif [ -r /proc/config.gz -a ! -z "$CFG_ZCAT" ]
+        then
+            if "$CFG_ZCAT" /proc/config.gz | grep --quiet "CONFIG_GRKERNSEC=y"
+            then
+                GRSEC_DETECTED=1
+            fi
+        # Flaky.
+        elif grep --quiet grsec /proc/version
+        then
+            GRSEC_DETECTED=1
+        fi
+
+        if [ ! -z "$GRSEC_DETECTED" ]
+        then
+            step_msg "GRSecurity: yes"
+            if [ ! -z "$CFG_PAXCTL" ]
+            then
+                CFG_ENABLE_PAX_FLAGS=1
+            else
+                warn "GRSecurity kernel detected but no paxctl binary found: not setting CFG_ENABLE_PAX_FLAGS"
+            fi
+        else
+            step_msg "GRSecurity: no"
+        fi
     fi
 fi
 
@@ -523,15 +590,23 @@ then
     msg "git: submodule sync"
     "${CFG_GIT}" submodule --quiet sync
 
+    msg "git: submodule update"
+    "${CFG_GIT}" submodule --quiet update --init
+    need_ok "git failed"
+
+    msg "git: submodule foreach sync"
+    "${CFG_GIT}" submodule --quiet foreach --recursive 'if test -e .gitmodules; then git submodule sync; fi'
+    need_ok "git failed"
+
+    msg "git: submodule foreach update"
+    "${CFG_GIT}" submodule --quiet update --init --recursive
+    need_ok "git failed"
+
     # NB: this is just for the sake of getting the submodule SHA1 values
     # and status written into the build log.
     msg "git: submodule status"
     "${CFG_GIT}" submodule status --recursive
 
-    msg "git: submodule update"
-    "${CFG_GIT}" submodule --quiet update --init --recursive
-    need_ok "git failed"
-
     msg "git: submodule clobber"
     "${CFG_GIT}" submodule --quiet foreach --recursive git clean -dxf
     need_ok "git failed"
@@ -699,6 +774,12 @@ putvar CFG_C_COMPILER
 putvar CFG_LIBDIR
 putvar CFG_DISABLE_MANAGE_SUBMODULES
 
+if [ ! -z "$CFG_ENABLE_PAX_FLAGS" ]
+then
+    putvar CFG_ENABLE_PAX_FLAGS
+    putvar CFG_PAXCTL
+fi
+
 if [ ! -z $BAD_PANDOC ]
 then
     CFG_PANDOC=

doc/lib/codemirror-rust.js

@@ -9,7 +9,7 @@ CodeMirror.defineMode("rust", function() {
     "as": "op", "true": "atom", "false": "atom", "assert": "op", "check": "op",
     "claim": "op", "extern": "ignore", "unsafe": "ignore", "import": "else-style",
     "export": "else-style", "copy": "op", "log": "op",
-    "use": "op", "self": "atom"
+    "use": "op", "self": "atom", "pub": "atom", "priv": "atom"
   };
   var typeKeywords = function() {
     var keywords = {"fn": "fn"};

doc/prep.js

@@ -60,9 +60,10 @@ while ((line = lines[cur++]) != null) {
       var html = '<pre class="cm-s-default">', curstr = "", curstyle = null;
       function add(str, style) {
         if (style != curstyle) {
-          if (curstyle) html += '<span class="cm-' + curstyle + '">' + curstr
-            + "</span>";
-          else if (curstr) html += curstr;
+          if (curstyle) html +=
+            '<span class="cm-' + CodeMirror.htmlEscape(curstyle) + '">' +
+            CodeMirror.htmlEscape(curstr) + "</span>";
+          else if (curstr) html += CodeMirror.htmlEscape(curstr);
           curstr = str; curstyle = style;
         } else curstr += str;
       }